senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Scraped data

Browse Source
This commit is contained in:
Senad Uka
2020-09-10 19:46:25 +02:00
parent 70779b24c0
commit 855b93ca41
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/crawler/specificCrawlers/olx.js
View File

@@ -222,16 +222,17 @@ class OlxCrawler {
const asyncScraping = []; const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) { for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefs[i])); asyncScraping.push(hrefs[i]);
} }
const { scrapedData, errors } = await PromisePool const { scrapedData, errors } = await PromisePool
.withConcurrency(2) .withConcurrency(3)
.for(asyncScraping) .for(asyncScraping)
.process(async data => { .process(async data => {
return await data const result = await this.scrapeAd(data)
}) return result;
})
const filteredScrapedData = scrapedData.filter(adData => !!adData); const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData; return filteredScrapedData;