From 855b93ca41be569a6416b9f6d220a7381d69ac03 Mon Sep 17 00:00:00 2001 From: Senad Uka Date: Thu, 10 Sep 2020 19:46:25 +0200 Subject: [PATCH] Scraped data --- app/crawler/specificCrawlers/olx.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/crawler/specificCrawlers/olx.js b/app/crawler/specificCrawlers/olx.js index b4e8149..a979089 100644 --- a/app/crawler/specificCrawlers/olx.js +++ b/app/crawler/specificCrawlers/olx.js @@ -222,16 +222,17 @@ class OlxCrawler { const asyncScraping = []; for (let i = 0; i < actualNoOfResults; i++) { - asyncScraping.push(this.scrapeAd(hrefs[i])); + asyncScraping.push(hrefs[i]); } const { scrapedData, errors } = await PromisePool - .withConcurrency(2) + .withConcurrency(3) .for(asyncScraping) .process(async data => { - return await data - }) + const result = await this.scrapeAd(data) + return result; + }) const filteredScrapedData = scrapedData.filter(adData => !!adData); return filteredScrapedData;