senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Fixed crawler not reading and comparing all RERequest results

Browse Source
This commit is contained in:
Nedim Uka
2019-07-12 16:13:03 +02:00
parent 4517624fa8
commit 753a09aa36
5 changed files with 46 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/helpers/crawlers/olxClawler.js
View File

@@ -36,10 +36,13 @@ module.exports = class OlxCrawler {
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
console.log("Final crawler results");
if (results[0]) {
console.log(results[0].length);
// console.log(results);
const flatResults = results.flat();
console.log(flatResults);
if (flatResults) {
console.log(flatResults.length);
for (const finalResult of results[0]) {
for (const finalResult of flatResults) {
if (null !== finalResult) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {