senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'send-notification' into 'master'

Browse Source 
Send notification

See merge request saburly/marketalarm/web!18
This commit was merged in pull request #18.
This commit is contained in:
Bilal Catic
2019-07-02 10:32:11 +00:00
parent 5ffdaef1bf 8b20f0e170
commit 7f0b2d299e
10 changed files with 269 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
app/helpers/awsEmail.js
View File

@@ -1,9 +1,10 @@
const dotenv = require('dotenv');
dotenv.config();
const dotenv = require('dotenv').config();
const { getRealEstateTypeEnum } = require('./enums');
const { getRegionName, getMunicipalityName } = require('./codes');
const AWS = require('aws-sdk');
var AWS = require('aws-sdk');
const TEMPLATE_NAME = "MarketAlertTemplate"
AWS.config.update({
region: process.env.AMAZON_REGION,
credentials:
@@ -27,11 +28,11 @@ const sendTemplatedEmail = async (email, request) => {
Body: { /* required */
Html: {
Charset: "UTF-8",
Data: getEmailHTML(request)
Data: getGreetingsEmailHTML(request)
},
Text: {
Charset: "UTF-8",
Data: getEmaiTextVersion(request)
Data: getGreetingsEmaiTextVersion(request)
}
},
Subject: {
@@ -49,7 +50,7 @@ const sendTemplatedEmail = async (email, request) => {
await sendEmailPromise;
}
const getEmailHTML = (realestateRequest) => {
const getGreetingsEmailHTML = (realestateRequest) => {
const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2 </strong></div>` : ``
@@ -75,15 +76,15 @@ Javimi tim.
}
const getEmaiTextVersion = (realestateRequest) => {
const getGreetingsEmaiTextVersion = (realestateRequest) => {
const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
+ realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
+ "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
+ gardenSize
"\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
"\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
"\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
"\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
"\n Tvoj,\n Javimi tim"
@@ -91,6 +92,111 @@ const getEmaiTextVersion = (realestateRequest) => {
return text;
}
const sendBulkEmail = async (marketAlerts) => {
try {
destinations = []
groupedEmails = [];
marketAlerts.forEach(marketAlert => {
if (!groupedEmails[marketAlert.email]) {
groupedEmails[marketAlert.email] = [];
groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
} else {
groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
}
});
for (email in groupedEmails) {
const url = groupedEmails[email];
let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
destinations.push({
Destination: {
ToAddresses: [
email
]
},
ReplacementTemplateData: repData
})
}
console.log("AWS EMAIL : Bulk email replacement data:");
console.log(destinations);
var params = {
Destinations:
destinations,
Source: process.env.SOURCE_EMAIL, /* required */
Template: TEMPLATE_NAME, /* required */
DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
ReplyToAddresses: [
process.env.SOURCE_EMAIL,
]
};
// Create the promise and SES service object
const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
const awsResult = await sendPromise;
console.log("AWS SES bulk email response");
console.log(awsResult);
} catch (e) {
console.log("Could not send bulk email", e)
}
}
const toAWSArray = (urlArray) => {
let arrayString = ""
urlArray.forEach(element => {
const formatetdTitle = element.title.replace(/"/g, "");
arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
});
return arrayString.slice(0, -1);
}
const getNotificationEmailHtml = () => {
return `<h2> Zdravo,
Pronašli smo nekretninu koju ste tražili. </h2>
<h3> Ovo su tražene nekretnine: </h3>
<div>
<div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/>
<div/>
</div>`
}
const getNotificationEmailText = () => {
return ` Zdravo,
Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
}
const createMarketAlertEmailTemplate = async () => {
const marketAlertTemplate = {
Template: {
TemplateName: "MarketAlertTemplate",
SubjectPart: "Javi mi obavijest",
TextPart: getNotificationEmailText(),
HtmlPart: getNotificationEmailHtml()
}
}
try {
const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
await templatePromise
} catch (e) {
console.log("Could not create MarketAlertEmailTemplate", e);
}
}
module.exports = {
sendTemplatedEmail
sendTemplatedEmail,
sendBulkEmail,
createMarketAlertEmailTemplate
};

29
app/helpers/crawlers/olxClawler.js
View File

@@ -27,7 +27,7 @@ module.exports = class OlxCrawler {
// }
//TODO remove properties that are not needed, and add some if they are missing
const title = $('#naslovartikla').text();
const title = $('#naslovartikla').text().trim();
const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
const price = $('#pc > p:nth-child(2)').text();
@@ -87,14 +87,14 @@ module.exports = class OlxCrawler {
const data = {
realEstateType: this.getCategoryId(realEstateType),
email : email,
email: email,
olxId: olxId,
// category: category,
url,
title,
price: isNaN(parsedPrice) ? 0 : parsedPrice,
size: parseFloat(size),
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
address,
region,
municipality,
@@ -151,22 +151,22 @@ module.exports = class OlxCrawler {
}
}
getCategoryId (category) {
getCategoryId(category) {
switch(category) {
switch (category) {
case 'Stanovi':
return 'stan';
return 'stan';
case 'Vikendice':
return 'vikendica'
return 'vikendica'
case 'Kuće':
return 'kuca';
default:
return '';
}
}
return '';
}
}
async indexPages(urls, start, end, maxResults = 1000) {
//TODO fix paging
@@ -186,16 +186,18 @@ module.exports = class OlxCrawler {
}
async crawl() {
console.log("OLX CRAWLER: start crawl");
const filteredResults = [];
const realestateRequests = await allRERequest()
const realestateRequests = await allRERequest();
console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
for (const result of results) {
for (const finalResult of result) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]);
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
@@ -203,8 +205,7 @@ module.exports = class OlxCrawler {
}
}
}
console.log(filteredResults);
console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
return filteredResults;
}

47
app/helpers/db/dbHelper.js
View File

@@ -1,15 +1,54 @@
const db = require('../../models/index');
// TODO Fetch only subscribed realestate requests
/**
* Find all subscribed RealEstateRequests
*/
const allRERequest = async () => {
return await db.RealEstateRequest.findAll();
return await db.RealEstateRequest.findAll({
where: {
subscribed: true
}
});
}
const findPointInsideBoundingBox = async (latLng) => {
return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
/**
* Find all , or all depending on notified bolean marketalerts, and order them by email
*
* @param fechAll bolean
* @param notified bolean
*
* @returns array of MarketAlerts
*/
const allMarketAlerts = async (fetchAll, notified) => {
let queryObject = {
order: [
['email', 'DESC'],
]
}
if (!fetchAll){
queryObject.where = {
notified: notified
}
}
return await db.MarketAlert.findAll(queryObject);
}
/**
* Find all unnotified marketalerts
* @param latLng array
* @param email string
*
* @returns array of MarketAlerts
*/
const findPointInsideBoundingBox = async (latLng, email) => {
return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
}
module.exports = {
allRERequest,
allMarketAlerts,
findPointInsideBoundingBox
};

11
app/helpers/url.js
View File

@@ -7,17 +7,6 @@ const currentRERequest = async (req) => {
const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
return request;
};
// TODO Fetch only subscribed realestate requests
const allRERequest = async () => {
return await db.RealEstateRequest.findAll();
}
const findPointInsideBoundingBox = async (latLng) => {
return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
}
module.exports = {
currentRERequest,
allRERequest,
findPointInsideBoundingBox
};

20
app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js Normal file
View File

@@ -0,0 +1,20 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
'MarketAlerts',
'notified',
{
type: Sequelize.BOOLEAN
}
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn(
'MarketAlerts',
'notified'
);
}
};

20
app/migrations/20190628165512-add-title-to-marketalerts.js Normal file
View File

@@ -0,0 +1,20 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
'MarketAlerts',
'title',
{
type: Sequelize.STRING
}
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn(
'MarketAlerts',
'title'
);
}
};

2
app/models/marketalert.js
View File

@@ -11,6 +11,8 @@ module.exports = (sequelize, DataTypes) => {
municipality : DataTypes.STRING,
region : DataTypes.STRING,
realEstateType : DataTypes.STRING,
notified : DataTypes.BOOLEAN,
title : DataTypes.STRING,
email: {
type: DataTypes.STRING,

27
app/services/crawlerService.js
View File

@@ -2,6 +2,7 @@
const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');
const olxCrawler = new OlxCrawler(1, 2, 3);
@@ -10,6 +11,7 @@ const crawlers = [
];
async function crawlAll() {
console.log("CRAWLER SERVICE: crawlAll");
Promise.map(crawlers, function (crawler) {
return crawler.crawl();
@@ -17,7 +19,8 @@ async function crawlAll() {
try {
const marketAlertsFromDb = await db.MarketAlert.findAll();
const marketAlertsFromDb = await allMarketAlerts(true);
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
@@ -34,22 +37,30 @@ async function crawlAll() {
municipality: result.municipality,
region: result.region,
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
realEstateType: result.realEstateType
realEstateType: result.realEstateType,
title: result.title,
notified: false
})
}
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
try {
console.log(marketAlerts);
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
process.exit()
process.exit();
} catch (e) {
console.log("Could not bulkCreate marketalers reason: ", e);
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
process.exit();
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
process.exit();
}
})
};
crawlAll();

32
app/services/notificationService.js Normal file
View File

@@ -0,0 +1,32 @@
const Promise = require("bluebird");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');
const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
async function processNotifications() {
try {
const marketAlerts = await allMarketAlerts(false, false);
console.log(marketAlerts.length)
await createMarketAlertEmailTemplate();
if (marketAlerts.length > 0) {
console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
await sendBulkEmail(marketAlerts);
} else {
console.log("NOTIFICATION SERVICE: No new alerts");
return;
}
await db.MarketAlert.update(
{ notified: true }, /* set attributes' value */
{ where: { notified: false } } /* where criteria */
);
process.exit();
} catch (e) {
console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
}
}
processNotifications();

3
package.json
View File

@@ -7,7 +7,8 @@
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node ./index.js",
"start-mon": "nodemon ./index.js",
"scheduler": "node ./app/services/crawlerService.js",
"crawler": "node ./app/services/crawlerService.js",
"notification": "node ./app/services/notificationService.js",
"migrate": "cd app && npx sequelize db:migrate",
"setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
"docker-start": "docker start pg_marketalerts",