senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Filter data by geolocation now sets hasLocation boolean instead of excluding results

Browse Source
This commit is contained in:
Nedim Uka
2019-07-10 15:21:46 +02:00
parent 5829de64e0
commit 33f9e37d93
5 changed files with 64 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/helpers/crawlers/olxClawler.js
View File

@@ -45,7 +45,12 @@ module.exports = class OlxCrawler {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
finalResult.hasLocation = true
filteredResults.push(finalResult);
} else {
finalResult.hasLocation = false
filteredResults.push(finalResult);
}
}
@@ -270,6 +275,16 @@ class Indexer {
if (this.olxUrl.url === undefined) {
return {}
}
if (global.hrefs) {
if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
console.log("We found duplicate URL");
return null
}
}
const res = await fetch(this.olxUrl.url);
const body = await res.text();
const $ = cheerio.load(body);