senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Promise pool

Browse Source
This commit is contained in:
Senad Uka
2020-09-10 19:39:13 +02:00
parent 26377c485c
commit e4775158fc
4 changed files with 25 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/crawler/specificCrawlers/olx.js
View File

@@ -4,6 +4,7 @@ const fetch = require("../../helpers/fetchWrapper");
const cheerio = require("cheerio"); const cheerio = require("cheerio");
const Promise = require("bluebird"); const Promise = require("bluebird");
const moment = require("moment-timezone"); const moment = require("moment-timezone");
const PromisePool = require('@supercharge/promise-pool');
const { const {
AD_TYPE, AD_TYPE,
@@ -224,6 +225,14 @@ class OlxCrawler {
asyncScraping.push(this.scrapeAd(hrefs[i])); asyncScraping.push(this.scrapeAd(hrefs[i]));
} }
const { scrapedData, errors } = await PromisePool
.withConcurrency(2)
.for(asyncScraping)
.process(async data => {
return await data
})
const scrapedData = await Promise.all(asyncScraping); const scrapedData = await Promise.all(asyncScraping);
const filteredScrapedData = scrapedData.filter(adData => !!adData); const filteredScrapedData = scrapedData.filter(adData => !!adData);
return filteredScrapedData; return filteredScrapedData;

2
app/helpers/fetchWrapper.js
View File

@@ -25,8 +25,6 @@ const fetch = async (url, options = {}) => {
const urlAdaptedForScraping = USE_SCRAPER_API const urlAdaptedForScraping = USE_SCRAPER_API
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}` ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
: url; : url;
const randomPauseMS = Math.floor(Math.random() * Math.floor(1500)) + 500;
await timeout(randomPauseMS);
return nodeFetch(urlAdaptedForScraping, newOptions); return nodeFetch(urlAdaptedForScraping, newOptions);
}; };

15
package-lock.json generated
View File

@@ -40,6 +40,21 @@
"@sendgrid/helpers": "^6.3.0" "@sendgrid/helpers": "^6.3.0"
} }
}, },
"@supercharge/goodies": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
"integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g==",
"dev": true
},
"@supercharge/promise-pool": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
"integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
"dev": true,
"requires": {
"@supercharge/goodies": "~1.4.0"
}
},
"@types/caseless": { "@types/caseless": {
"version": "0.12.2", "version": "0.12.2",
"resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz",

1
package.json
View File

@@ -55,6 +55,7 @@
"sequelize-cli": "^5.5.0" "sequelize-cli": "^5.5.0"
}, },
"devDependencies": { "devDependencies": {
"@supercharge/promise-pool": "^1.3.0",
"nodemon": "^1.19.0" "nodemon": "^1.19.0"
} }
} }