senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added node schedule to run crawler and notification service

Browse Source
This commit is contained in:
Nedim Uka
2019-07-09 16:33:00 +02:00
parent 1b098f181c
commit a43723485c
6 changed files with 261 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/helpers/crawlers/olxClawler.js
View File

@@ -36,23 +36,26 @@ module.exports = class OlxCrawler {
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
console.log("Final crawler results");
console.log(results[0].length);
if (results[0]) {
console.log(results[0].length);
for (const finalResult of results[0]) {
for (const finalResult of results[0]) {
if (null !== finalResult) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (null !== finalResult) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
}
}
}
}
}
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
}
return []
}
createRequestUrls(realestateRequests) {

9
app/services/crawlerService.js
View File

@@ -13,7 +13,7 @@ const crawlers = [
async function crawlAll() {
console.log("CRAWLER SERVICE: crawlAll");
Promise.map(crawlers, function (crawler) {
return Promise.map(crawlers, function (crawler) {
return crawler.crawl();
}).then(async (results) => {
@@ -51,17 +51,14 @@ async function crawlAll() {
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
process.exit();
} catch (e) {
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
process.exit();
}
} catch (e) {
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
process.exit();
}
})
};
crawlAll();
module.exports = crawlAll;
// crawlAll();

6
app/services/notificationService.js
View File

@@ -1,5 +1,4 @@
const Promise = require("bluebird");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');
const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
@@ -16,18 +15,15 @@ async function processNotifications() {
await sendBulkEmail(marketAlerts);
} else {
console.log("NOTIFICATION SERVICE: No new alerts");
process.exit();
}
await db.MarketAlert.update(
{ notified: true }, /* set attributes' value */
{ where: { notified: false } } /* where criteria */
);
process.exit();
} catch (e) {
console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
process.exit();
}
}
processNotifications();
module.exports = processNotifications;