senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Reduced pager to 5 pages at a time

Browse Source
This commit is contained in:
Nedim Uka
2019-07-08 13:02:28 +02:00
parent 2dd1eaa5fd
commit 1b098f181c
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/helpers/crawlers/olxClawler.js
View File

@@ -134,7 +134,7 @@ class Indexer {
} else { } else {
console.log("HAS MORE RESULTS, should only contain HREFS"); console.log("HAS MORE RESULTS, should only contain HREFS");
console.log(results.length); console.log(results.length);
const newResults = await this.indexWithPagination(results[0].pageNumber + 10); const newResults = await this.indexWithPagination(results[0].pageNumber + 5);
const singlePageIndexers = this.prepareHrefIndexers(results); const singlePageIndexers = this.prepareHrefIndexers(results);
const newerResults = await Promise.map(singlePageIndexers, function (indexer) { const newerResults = await Promise.map(singlePageIndexers, function (indexer) {
@@ -161,7 +161,7 @@ class Indexer {
const indexers = []; const indexers = [];
let lastPageNumber; let lastPageNumber;
if (pageNr) { if (pageNr) {
for (let index = Number(pageNr[0]); index <= Number(pageNr[0]) + 10; index++) { for (let index = Number(pageNr[0]); index <= Number(pageNr[0]) + 5; index++) {
lastPageNumber = index; lastPageNumber = index;
const newOlxUrl = { const newOlxUrl = {
url: this.olxUrl.url.replace(/\d+$/, "") + index, url: this.olxUrl.url.replace(/\d+$/, "") + index,
@@ -172,7 +172,7 @@ class Indexer {
} }
} else { } else {
for (let index = 1; index <= 10; index++) { for (let index = 1; index <= 5; index++) {
lastPageNumber = index; lastPageNumber = index;
const newOlxUrl = { const newOlxUrl = {
url: this.olxUrl.url + index, url: this.olxUrl.url + index,