Log for debugging of concurrent req.

2020-03-05 18:05:37 +01:00
parent 41c926b5bb
commit 824414adad
1 changed files with 8 additions and 1 deletions
--- a/app/crawler/specificCrawlers/olx.js
+++ b/app/crawler/specificCrawlers/olx.js
@@ -201,10 +201,17 @@ class OlxCrawler {
          i,
          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
        );
-
+        //
+        console.log(
+          `OLX - Sending requests from ${i} to ${i +
+            NUMBER_OF_CONCURRENT_REQ_SCRAPER_API}.`
+        );
+        console.log(`OLX - Urls sent to scrape: `, concurrentUrlsToScrape);
+        //
        const concurrentReqScraperApi = concurrentUrlsToScrape.map(url =>
          this.scrapeAd(url)
        );
+
        const concurrentReqData = await Promise.all(concurrentReqScraperApi);

        concurrentReqData.forEach(reqData => scrapedData.push(reqData));