diff --git a/app/crawler/crawl.js b/app/crawler/crawl.js index 86f894c..9aafc75 100644 --- a/app/crawler/crawl.js +++ b/app/crawler/crawl.js @@ -10,6 +10,7 @@ const RentalCrawler = require("./specificCrawlers/rental"); const ProstorCrawler = require("./specificCrawlers/prostor"); const AktidoCrawler = require("./specificCrawlers/aktido"); const SaljicCrawler = require("./specificCrawlers/saljic"); +const { logDebug } = require("../helpers/log"); const { OLX_CONFIG, @@ -32,34 +33,34 @@ async function crawlAll() { OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE, OLX_CONFIG.OLX_IGNORED_USERNAMES, OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES - ), - new RentalCrawler( - [postgresSaver], - RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE, - RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES, - RENTAL_CONFIG.RENTAL_MAX_PAGES, - RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE, - RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES, - RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES - ), - new ProstorCrawler( - [postgresSaver], - PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE, - PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES, - PROSTOR_CONFIG.PROSTOR_MAX_PAGES, - PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE, - PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES, - PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES - ), - new AktidoCrawler( - [postgresSaver], - AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE, - AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES, - AKTIDO_CONFIG.AKTIDO_MAX_PAGES, - AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE, - AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES, - AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES ) + //new RentalCrawler( + //[postgresSaver], + //RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE, + //RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES, + //RENTAL_CONFIG.RENTAL_MAX_PAGES, + //RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE, + //RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES, + //RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES + //), + //new ProstorCrawler( + //[postgresSaver], + //PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE, + //PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES, + //PROSTOR_CONFIG.PROSTOR_MAX_PAGES, + //PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE, + //PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES, + //PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES + //), + //new AktidoCrawler( + //[postgresSaver], + //AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE, + //AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES, + //AKTIDO_CONFIG.AKTIDO_MAX_PAGES, + //AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE, + //AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES, + //AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES + //) //, //new SaljicCrawler( //[postgresSaver], @@ -76,7 +77,9 @@ async function crawlAll() { for (const crawler of crawlers) { try { + logDebug('Starting crawler: ', crawler); const newRealEstatesFromSingleCrawler = await crawler.crawl(); + logDebug('Crawler done: ', crawler); if (Array.isArray(newRealEstatesFromSingleCrawler)) { newRealEstates.push(...newRealEstatesFromSingleCrawler); } diff --git a/index.js b/index.js index c32882b..45469b2 100644 --- a/index.js +++ b/index.js @@ -39,7 +39,7 @@ app.listen(APP_PORT, () => let crawlerRunning = STOP_CRAWLER; const crawl = () => { - logDebug("Crawl start"); + logDebug("Crawl start. crawlerRunning: ", crawlerRunning); if (!crawlerRunning) { crawlerRunning = true; crawlAll().then(newRealEstates => {