senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Log for debugging of concurrent req.

Browse Source
This commit is contained in:
Naida Vatric
2020-03-05 18:05:37 +01:00
parent 41c926b5bb
commit 824414adad
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/crawler/specificCrawlers/olx.js
View File

@@ -201,10 +201,17 @@ class OlxCrawler {
i, i,
i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
); );
//
console.log(
`OLX - Sending requests from ${i} to ${i +
NUMBER_OF_CONCURRENT_REQ_SCRAPER_API}.`
);
console.log(`OLX - Urls sent to scrape: `, concurrentUrlsToScrape);
//
const concurrentReqScraperApi = concurrentUrlsToScrape.map(url => const concurrentReqScraperApi = concurrentUrlsToScrape.map(url =>
this.scrapeAd(url) this.scrapeAd(url)
); );
const concurrentReqData = await Promise.all(concurrentReqScraperApi); const concurrentReqData = await Promise.all(concurrentReqScraperApi);
concurrentReqData.forEach(reqData => scrapedData.push(reqData)); concurrentReqData.forEach(reqData => scrapedData.push(reqData));