moar fixes

This commit is contained in:
=
2020-09-10 14:26:48 -07:00
parent 160efdf6ab
commit e8115a9215

View File

@@ -228,16 +228,18 @@ class OlxCrawler {
}
const hamo = []
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.for(asyncScraping)
.process(async data => {
const result = await this.scrapeAd(data)
await this.sleep(this.delayBetweenPages);
return result;
hamo.push(result)
return result; //TODO: this does not work, scrapedData is null, hamo works
})
const filteredScrapedData = scrapedData.filter(adData => !!adData);
const filteredScrapedData = hamo.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("Exception caught:" + e);