senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Fiks

Browse Source
This commit is contained in:
=
2020-09-10 13:31:57 -07:00
parent 855b93ca41
commit c9b8c2e1a5
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/crawler/specificCrawlers/olx.js
View File

@@ -178,6 +178,7 @@ class OlxCrawler {
this.maxResultsPerPage this.maxResultsPerPage
); );
await this.sleep(this.delayBetweenPages);
if (Array.isArray(singlePageResults) && singlePageResults.length > 0) { if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
yield singlePageResults; yield singlePageResults;
} else { } else {
@@ -227,10 +228,11 @@ class OlxCrawler {
const { scrapedData, errors } = await PromisePool const { scrapedData, errors } = await PromisePool
.withConcurrency(3) .withConcurrency(2)
.for(asyncScraping) .for(asyncScraping)
.process(async data => { .process(async data => {
const result = await this.scrapeAd(data) const result = await this.scrapeAd(data)
await this.sleep(this.delayBetweenPages);
return result; return result;
}) })
@@ -243,7 +245,7 @@ class OlxCrawler {
} }
async scrapeAd(url) { async scrapeAd(url) {
// console.log("Scraping : ", url); //console.log("Scraping : ", url);
try { try {
const adPageSource = await fetch(url); const adPageSource = await fetch(url);
const body = await adPageSource.text(); const body = await adPageSource.text();

4
package-lock.json generated
View File

@@ -43,14 +43,12 @@
"@supercharge/goodies": { "@supercharge/goodies": {
"version": "1.4.0", "version": "1.4.0",
"resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz", "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
"integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g==", "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
"dev": true
}, },
"@supercharge/promise-pool": { "@supercharge/promise-pool": {
"version": "1.3.0", "version": "1.3.0",
"resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz", "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
"integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==", "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
"dev": true,
"requires": { "requires": {
"@supercharge/goodies": "~1.4.0" "@supercharge/goodies": "~1.4.0"
} }