senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

moar fixes

Browse Source
This commit is contained in:
=
2020-09-10 14:26:48 -07:00
parent 160efdf6ab
commit e8115a9215
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/crawler/specificCrawlers/olx.js
View File

@@ -228,16 +228,18 @@ class OlxCrawler {
} }
const hamo = []
const { scrapedData, errors } = await PromisePool const { scrapedData, errors } = await PromisePool
.withConcurrency(2) .withConcurrency(2)
.for(asyncScraping) .for(asyncScraping)
.process(async data => { .process(async data => {
const result = await this.scrapeAd(data) const result = await this.scrapeAd(data)
await this.sleep(this.delayBetweenPages); await this.sleep(this.delayBetweenPages);
return result; hamo.push(result)
return result; //TODO: this does not work, scrapedData is null, hamo works
}) })
const filteredScrapedData = scrapedData.filter(adData => !!adData); const filteredScrapedData = hamo.filter(adData => !!adData);
return filteredScrapedData; return filteredScrapedData;
} catch (e) { } catch (e) {
console.error("Exception caught:" + e); console.error("Exception caught:" + e);