senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added send notification service, and queried unsent marketalerts, fixed some issues with crawler, and added proper logging

Browse Source
This commit is contained in:
Nedim Uka
2019-06-25 17:06:07 +02:00
parent 5ffdaef1bf
commit 208faa08df
8 changed files with 123 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
app/helpers/crawlers/olxClawler.js
View File

@@ -87,14 +87,14 @@ module.exports = class OlxCrawler {
const data = {
realEstateType: this.getCategoryId(realEstateType),
email : email,
email: email,
olxId: olxId,
// category: category,
url,
title,
price: isNaN(parsedPrice) ? 0 : parsedPrice,
size: parseFloat(size),
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
address,
region,
municipality,
@@ -151,22 +151,22 @@ module.exports = class OlxCrawler {
}
}
getCategoryId (category) {
getCategoryId(category) {
switch(category) {
switch (category) {
case 'Stanovi':
return 'stan';
return 'stan';
case 'Vikendice':
return 'vikendica'
return 'vikendica'
case 'Kuće':
return 'kuca';
default:
return '';
}
}
return '';
}
}
async indexPages(urls, start, end, maxResults = 1000) {
//TODO fix paging
@@ -186,16 +186,18 @@ module.exports = class OlxCrawler {
}
async crawl() {
console.log("OLX CRAWLER: start crawl");
const filteredResults = [];
const realestateRequests = await allRERequest()
const realestateRequests = await allRERequest();
console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
for (const result of results) {
for (const finalResult of result) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]);
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
@@ -203,8 +205,7 @@ module.exports = class OlxCrawler {
}
}
}
console.log(filteredResults);
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
}