senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Fix saljic

Browse Source
This commit is contained in:
Senad Uka
2020-09-11 04:51:54 +02:00
parent e8115a9215
commit d4fcd1950d
2 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/crawler/specificCrawlers/olx.js
View File

@@ -228,18 +228,18 @@ class OlxCrawler {
}
const hamo = []
const dataResults = []
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.for(asyncScraping)
.process(async data => {
const result = await this.scrapeAd(data)
await this.sleep(this.delayBetweenPages);
hamo.push(result)
return result; //TODO: this does not work, scrapedData is null, hamo works
dataResults.push(result)
return result; //TODO: this does not work, scrapedData is null, dataResults works
})
const filteredScrapedData = hamo.filter(adData => !!adData);
const filteredScrapedData = dataResults.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("Exception caught:" + e);

18
app/crawler/specificCrawlers/saljic.js
View File

@@ -207,11 +207,23 @@ class SaljicCrawler {
const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
asyncScraping.push([hrefsAbs[i], adTypes[i]]);
}
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
const dataResults = []
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.for(asyncScraping)
.process(async data => {
const result = await this.scrapeAd(...data)
await this.sleep(this.delayBetweenPages);
dataResults.push(result)
return result; //TODO: this does not work, scrapedData is null, dataResults works
})
const filteredScrapedData = dataResults.filter(adData => !!adData);
return filteredScrapedData;
} catch (e) {
console.error("[SALJIC] Exception caught:" + e);