+
{{#each marketAlertUrl}}
{{title}}{{/each}}
+
+
`
+}
+
+const getNotificationEmailText = () => {
+ return ` Zdravo,
+ Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
+}
+
+const createMarketAlertEmailTemplate = async () => {
+ const marketAlertTemplate = {
+ Template: {
+ TemplateName: "MarketAlertTemplate",
+ SubjectPart: "Javi mi obavijest",
+ TextPart: getNotificationEmailText(),
+ HtmlPart: getNotificationEmailHtml()
+ }
+ }
+
+ try {
+ const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
+ await templatePromise
+
+ } catch (e) {
+ console.log("Could not create MarketAlertEmailTemplate", e);
+ }
+}
+
module.exports = {
- sendTemplatedEmail
+ sendTemplatedEmail,
+ sendBulkEmail,
+ createMarketAlertEmailTemplate
};
diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js
index 1f7ea1b..556d9f3 100644
--- a/app/helpers/crawlers/olxClawler.js
+++ b/app/helpers/crawlers/olxClawler.js
@@ -27,7 +27,7 @@ module.exports = class OlxCrawler {
// }
//TODO remove properties that are not needed, and add some if they are missing
- const title = $('#naslovartikla').text();
+ const title = $('#naslovartikla').text().trim();
const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
const price = $('#pc > p:nth-child(2)').text();
@@ -87,14 +87,14 @@ module.exports = class OlxCrawler {
const data = {
realEstateType: this.getCategoryId(realEstateType),
- email : email,
+ email: email,
olxId: olxId,
// category: category,
url,
title,
price: isNaN(parsedPrice) ? 0 : parsedPrice,
size: parseFloat(size),
- gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
+ gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
address,
region,
municipality,
@@ -151,22 +151,22 @@ module.exports = class OlxCrawler {
}
}
- getCategoryId (category) {
+ getCategoryId(category) {
- switch(category) {
+ switch (category) {
case 'Stanovi':
- return 'stan';
+ return 'stan';
case 'Vikendice':
- return 'vikendica'
+ return 'vikendica'
case 'Kuće':
return 'kuca';
default:
- return '';
- }
- }
+ return '';
+ }
+ }
async indexPages(urls, start, end, maxResults = 1000) {
//TODO fix paging
@@ -186,16 +186,18 @@ module.exports = class OlxCrawler {
}
async crawl() {
+ console.log("OLX CRAWLER: start crawl");
const filteredResults = [];
- const realestateRequests = await allRERequest()
+ const realestateRequests = await allRERequest();
+ console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
for (const result of results) {
for (const finalResult of result) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
- const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]);
+ const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
@@ -203,8 +205,7 @@ module.exports = class OlxCrawler {
}
}
}
-
- console.log(filteredResults);
+ console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
}
diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js
index f51638b..3363e1b 100644
--- a/app/helpers/db/dbHelper.js
+++ b/app/helpers/db/dbHelper.js
@@ -1,15 +1,54 @@
const db = require('../../models/index');
-// TODO Fetch only subscribed realestate requests
+/**
+ * Find all subscribed RealEstateRequests
+ */
const allRERequest = async () => {
- return await db.RealEstateRequest.findAll();
+ return await db.RealEstateRequest.findAll({
+ where: {
+ subscribed: true
+ }
+ });
}
-const findPointInsideBoundingBox = async (latLng) => {
- return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+/**
+ * Find all , or all depending on notified bolean marketalerts, and order them by email
+ *
+ * @param fechAll bolean
+ * @param notified bolean
+ *
+ * @returns array of MarketAlerts
+ */
+const allMarketAlerts = async (fetchAll, notified) => {
+
+ let queryObject = {
+ order: [
+ ['email', 'DESC'],
+ ]
+ }
+
+ if (!fetchAll){
+ queryObject.where = {
+ notified: notified
+ }
+ }
+
+ return await db.MarketAlert.findAll(queryObject);
+ }
+
+/**
+ * Find all unnotified marketalerts
+ * @param latLng array
+ * @param email string
+ *
+ * @returns array of MarketAlerts
+ */
+const findPointInsideBoundingBox = async (latLng, email) => {
+ return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
}
module.exports = {
allRERequest,
+ allMarketAlerts,
findPointInsideBoundingBox
};
diff --git a/app/helpers/url.js b/app/helpers/url.js
index 854d2a7..8bf1cb5 100644
--- a/app/helpers/url.js
+++ b/app/helpers/url.js
@@ -7,17 +7,6 @@ const currentRERequest = async (req) => {
const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
return request;
};
-// TODO Fetch only subscribed realestate requests
-const allRERequest = async () => {
- return await db.RealEstateRequest.findAll();
-}
-
-const findPointInsideBoundingBox = async (latLng) => {
- return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
-}
-
module.exports = {
currentRERequest,
- allRERequest,
- findPointInsideBoundingBox
};
diff --git a/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js b/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js
new file mode 100644
index 0000000..f70e6f6
--- /dev/null
+++ b/app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js
@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+ up: (queryInterface, Sequelize) => {
+ return queryInterface.addColumn(
+ 'MarketAlerts',
+ 'notified',
+ {
+ type: Sequelize.BOOLEAN
+ }
+ );
+ },
+
+ down: (queryInterface, Sequelize) => {
+ return queryInterface.removeColumn(
+ 'MarketAlerts',
+ 'notified'
+ );
+ }
+};
diff --git a/app/migrations/20190628165512-add-title-to-marketalerts.js b/app/migrations/20190628165512-add-title-to-marketalerts.js
new file mode 100644
index 0000000..d874f3d
--- /dev/null
+++ b/app/migrations/20190628165512-add-title-to-marketalerts.js
@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+ up: (queryInterface, Sequelize) => {
+ return queryInterface.addColumn(
+ 'MarketAlerts',
+ 'title',
+ {
+ type: Sequelize.STRING
+ }
+ );
+ },
+
+ down: (queryInterface, Sequelize) => {
+ return queryInterface.removeColumn(
+ 'MarketAlerts',
+ 'title'
+ );
+ }
+};
diff --git a/app/models/marketalert.js b/app/models/marketalert.js
index 9f1e092..c23961b 100644
--- a/app/models/marketalert.js
+++ b/app/models/marketalert.js
@@ -11,6 +11,8 @@ module.exports = (sequelize, DataTypes) => {
municipality : DataTypes.STRING,
region : DataTypes.STRING,
realEstateType : DataTypes.STRING,
+ notified : DataTypes.BOOLEAN,
+ title : DataTypes.STRING,
email: {
type: DataTypes.STRING,
diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js
index 5eb160b..57d667a 100644
--- a/app/services/crawlerService.js
+++ b/app/services/crawlerService.js
@@ -2,6 +2,7 @@
const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
+const { allMarketAlerts } = require('../helpers/db/dbHelper');
const olxCrawler = new OlxCrawler(1, 2, 3);
@@ -10,6 +11,7 @@ const crawlers = [
];
async function crawlAll() {
+ console.log("CRAWLER SERVICE: crawlAll");
Promise.map(crawlers, function (crawler) {
return crawler.crawl();
@@ -17,7 +19,8 @@ async function crawlAll() {
try {
- const marketAlertsFromDb = await db.MarketAlert.findAll();
+ const marketAlertsFromDb = await allMarketAlerts(true);
+ console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
@@ -34,22 +37,30 @@ async function crawlAll() {
municipality: result.municipality,
region: result.region,
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
- realEstateType: result.realEstateType
+ realEstateType: result.realEstateType,
+ title: result.title,
+ notified: false
})
}
+ console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
+
try {
- console.log(marketAlerts);
- const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+
+ const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+ console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
+
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
- process.exit()
+ process.exit();
+
} catch (e) {
- console.log("Could not bulkCreate marketalers reason: ", e);
+ console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+ process.exit();
}
} catch (e) {
- console.log("Error crawling. Trying next crawler! ", e);
+ console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
+ process.exit();
}
})
};
crawlAll();
-
diff --git a/app/services/notificationService.js b/app/services/notificationService.js
new file mode 100644
index 0000000..f299330
--- /dev/null
+++ b/app/services/notificationService.js
@@ -0,0 +1,32 @@
+
+const Promise = require("bluebird");
+const db = require("../models/index");
+const { allMarketAlerts } = require('../helpers/db/dbHelper');
+const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
+
+
+async function processNotifications() {
+
+ try {
+ const marketAlerts = await allMarketAlerts(false, false);
+ console.log(marketAlerts.length)
+ await createMarketAlertEmailTemplate();
+ if (marketAlerts.length > 0) {
+ console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
+ await sendBulkEmail(marketAlerts);
+ } else {
+ console.log("NOTIFICATION SERVICE: No new alerts");
+ return;
+ }
+
+ await db.MarketAlert.update(
+ { notified: true }, /* set attributes' value */
+ { where: { notified: false } } /* where criteria */
+ );
+ process.exit();
+ } catch (e) {
+ console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
+ }
+}
+
+processNotifications();
diff --git a/package.json b/package.json
index 360b7bd..f7a3e66 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,8 @@
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node ./index.js",
"start-mon": "nodemon ./index.js",
- "scheduler": "node ./app/services/crawlerService.js",
+ "crawler": "node ./app/services/crawlerService.js",
+ "notification": "node ./app/services/notificationService.js",
"migrate": "cd app && npx sequelize db:migrate",
"setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
"docker-start": "docker start pg_marketalerts",