stop crawling when existing, not renewed ad is found
This commit is contained in:
@@ -18,18 +18,23 @@ const crawlers = [
|
||||
OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
|
||||
OLX_CONFIG.OLX_MAX_PAGES,
|
||||
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
|
||||
OLX_CONFIG.OLX_IGNORED_USERNAMES
|
||||
OLX_CONFIG.OLX_IGNORED_USERNAMES,
|
||||
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
|
||||
)
|
||||
];
|
||||
|
||||
async function crawlAll() {
|
||||
for (let crawler of crawlers) {
|
||||
try {
|
||||
await crawler.crawl();
|
||||
const newRealEstates = await crawler.crawl();
|
||||
|
||||
console.log("Number of new real estates : ", newRealEstates.length);
|
||||
} catch (e) {
|
||||
console.log("Error crawling. Trying next crawler! ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
crawlAll();
|
||||
(async () => {
|
||||
await crawlAll();
|
||||
})();
|
||||
|
||||
Reference in New Issue
Block a user