diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js index d36f835..75293f2 100644 --- a/app/crawler/specificCrawlers/olx.js +++ b/app/crawler/specificCrawlers/olx.js @@ -201,10 +201,17 @@ class OlxCrawler { i, i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API ); - + // + console.log( + `OLX - Sending requests from ${i} to ${i + + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API}.` + ); + console.log(`OLX - Urls sent to scrape: `, concurrentUrlsToScrape); + // const concurrentReqScraperApi = concurrentUrlsToScrape.map(url => this.scrapeAd(url) ); + const concurrentReqData = await Promise.all(concurrentReqScraperApi); concurrentReqData.forEach(reqData => scrapedData.push(reqData));