Scraped data
This commit is contained in:
@@ -222,16 +222,17 @@ class OlxCrawler {
|
||||
|
||||
const asyncScraping = [];
|
||||
for (let i = 0; i < actualNoOfResults; i++) {
|
||||
asyncScraping.push(this.scrapeAd(hrefs[i]));
|
||||
asyncScraping.push(hrefs[i]);
|
||||
}
|
||||
|
||||
|
||||
const { scrapedData, errors } = await PromisePool
|
||||
.withConcurrency(2)
|
||||
.withConcurrency(3)
|
||||
.for(asyncScraping)
|
||||
.process(async data => {
|
||||
return await data
|
||||
})
|
||||
const result = await this.scrapeAd(data)
|
||||
return result;
|
||||
})
|
||||
|
||||
const filteredScrapedData = scrapedData.filter(adData => !!adData);
|
||||
return filteredScrapedData;
|
||||
|
||||
Reference in New Issue
Block a user