Scraped data

This commit is contained in:
Senad Uka
2020-09-10 19:46:25 +02:00
parent 70779b24c0
commit 855b93ca41

View File

@@ -222,16 +222,17 @@ class OlxCrawler {
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i]));
asyncScraping.push(hrefs[i]);
}
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.withConcurrency(3)
.for(asyncScraping)
.process(async data => {
return await data
})
const result = await this.scrapeAd(data)
return result;
})
const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData;