senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added node schedule to run crawler and notification service

Browse Source
This commit is contained in:
Nedim Uka
2019-07-09 16:33:00 +02:00
parent 1b098f181c
commit a43723485c
6 changed files with 261 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/helpers/crawlers/olxClawler.js
View File

@@ -36,23 +36,26 @@ module.exports = class OlxCrawler {
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
console.log("Final crawler results");
console.log(results[0].length);
if (results[0]) {
console.log(results[0].length);
for (const finalResult of results[0]) {
for (const finalResult of results[0]) {
if (null !== finalResult) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (null !== finalResult) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
}
}
}
}
}
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
}
return []
}
createRequestUrls(realestateRequests) {