senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Preparing to save results to db

Browse Source
This commit is contained in:
Nedim Uka
2019-06-20 14:51:14 +02:00
parent c8ee848f0e
commit 1bcc5e8e5d
5 changed files with 40 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/helpers/crawlers/olxClawler.js
View File

@@ -110,8 +110,6 @@ module.exports = class OlxCrawler {
return null;
}
async indexPage(url, maxResults = 1000) {
try {
// console.log('Starting to index page: ' + pageNr);
@@ -138,7 +136,7 @@ module.exports = class OlxCrawler {
if (singleData) {
results.push(singleData);
}
await this.sleep(500);
// await this.sleep(500);
}
return results;
@@ -177,7 +175,7 @@ module.exports = class OlxCrawler {
let result = await this.indexPage(url, maxResults);
// Object.assign(results, result)
results.push(result);
await this.sleep(5000);
// await this.sleep(5000);
}
return results;
}