senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Fix saljic

Browse Source
This commit is contained in:
Senad Uka
2020-09-11 04:51:54 +02:00
parent e8115a9215
commit d4fcd1950d
2 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/crawler/specificCrawlers/olx.js
View File

@@ -228,18 +228,18 @@ class OlxCrawler {
} }
const hamo = [] const dataResults = []
const { scrapedData, errors } = await PromisePool const { scrapedData, errors } = await PromisePool
.withConcurrency(2) .withConcurrency(2)
.for(asyncScraping) .for(asyncScraping)
.process(async data => { .process(async data => {
const result = await this.scrapeAd(data) const result = await this.scrapeAd(data)
await this.sleep(this.delayBetweenPages); await this.sleep(this.delayBetweenPages);
hamo.push(result) dataResults.push(result)
return result; //TODO: this does not work, scrapedData is null, hamo works return result; //TODO: this does not work, scrapedData is null, dataResults works
}) })
const filteredScrapedData = hamo.filter(adData => !!adData); const filteredScrapedData = dataResults.filter(adData => !!adData);
return filteredScrapedData; return filteredScrapedData;
} catch (e) { } catch (e) {
console.error("Exception caught:" + e); console.error("Exception caught:" + e);

18
app/crawler/specificCrawlers/saljic.js
View File

@@ -207,11 +207,23 @@ class SaljicCrawler {
const asyncScraping = []; const asyncScraping = [];
for (let i = 0; i < actualNoOfResults; i++) { for (let i = 0; i < actualNoOfResults; i++) {
asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i])); asyncScraping.push([hrefsAbs[i], adTypes[i]]);
} }
const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData);
const dataResults = []
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.for(asyncScraping)
.process(async data => {
const result = await this.scrapeAd(...data)
await this.sleep(this.delayBetweenPages);
dataResults.push(result)
return result; //TODO: this does not work, scrapedData is null, dataResults works
})
const filteredScrapedData = dataResults.filter(adData => !!adData);
return filteredScrapedData; return filteredScrapedData;
} catch (e) { } catch (e) {
console.error("[SALJIC] Exception caught:" + e); console.error("[SALJIC] Exception caught:" + e);