stop crawling when existing, not renewed ad is found

This commit is contained in:
Bilal Catic
2019-09-25 08:54:33 +02:00
parent b3fcc6ba9a
commit c9a959f8be
2 changed files with 54 additions and 44 deletions

View File

@@ -18,18 +18,23 @@ const crawlers = [
OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
OLX_CONFIG.OLX_MAX_PAGES,
OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
OLX_CONFIG.OLX_IGNORED_USERNAMES
OLX_CONFIG.OLX_IGNORED_USERNAMES,
OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
)
];
async function crawlAll() {
for (let crawler of crawlers) {
try {
await crawler.crawl();
const newRealEstates = await crawler.crawl();
console.log("Number of new real estates : ", newRealEstates.length);
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
}
}
crawlAll();
(async () => {
await crawlAll();
})();