senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'notification-email-subject' into 'master'

Browse Source 
Notification email subject

See merge request saburly/marketalarm/web!19
This commit was merged in pull request #19.
This commit is contained in:
Bilal Catic
2019-07-04 07:44:49 +00:00
parent 7f0b2d299e a807cb5bf2
commit e4b3e3961d
8 changed files with 115 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -34,3 +34,10 @@ this will create and run postgres image and then execute migrations
5. Run app
`npm start` or `npm run start-mon` to run app with automatic restart on code change
### AWS SES
- AWS SES credentials are handled with env vratiables
- Notification emails are sent in batches of 50, by using SES templates
- Make sure that you are using different templates for different envirorments

82
app/helpers/awsEmail.js
View File

@@ -2,8 +2,10 @@
const dotenv = require('dotenv').config();
const { getRealEstateTypeEnum } = require('./enums');
const { getRegionName, getMunicipalityName } = require('./codes');
const db = require('../models/index');
const { allRERequestByUiid } = require('./db/dbHelper');
var AWS = require('aws-sdk');
const TEMPLATE_NAME = "MarketAlertTemplate"
const TEMPLATE_NAME = "MarketAlertTemplateDevelop"
AWS.config.update({
region: process.env.AMAZON_REGION,
@@ -37,7 +39,7 @@ const sendTemplatedEmail = async (email, request) => {
},
Subject: {
Charset: 'UTF-8',
Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}`
Data: `Javimi Potvrda: ${getSubject(request.realEstateType, request.region, request.municipality)}`
}
},
Source: process.env.SOURCE_EMAIL, /* required */
@@ -97,26 +99,65 @@ const sendBulkEmail = async (marketAlerts) => {
try {
destinations = []
groupedEmails = [];
groupedRERequests = [];
marketAlerts.forEach(marketAlert => {
if (!groupedEmails[marketAlert.email]) {
groupedEmails[marketAlert.email] = [];
groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
} else {
groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
}
let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request);
RERequestUuids = Array.from(new Set(RERequestUuids));
RERequestUuids = RERequestUuids.map(marketAlert => {
return { uniqueId: marketAlert }
});
for (email in groupedEmails) {
const RERequest = await allRERequestByUiid(RERequestUuids);
const requestDataValues = [];
const url = groupedEmails[email];
let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
RERequest.forEach(RERequest => {
var formatedRequest = {};
formatedRequest[RERequest.uniqueId] =
requestDataValues[RERequest.uniqueId] = {
realEstateType: RERequest.realEstateType,
region: RERequest.region,
municipality: RERequest.municipality
};
});
marketAlerts.forEach(marketAlert => {
const requestObject = {
email: marketAlert.email,
realEstateType: requestDataValues[marketAlert.request].realEstateType,
municipality: requestDataValues[marketAlert.request].municipality,
region: requestDataValues[marketAlert.request].region,
}
if (!groupedRERequests[marketAlert.request]) {
groupedRERequests[marketAlert.request] = {
requestObject: requestObject,
marketAlertArray: []
};
}
groupedRERequests[marketAlert.request].marketAlertArray.push({
url: marketAlert.url,
title: marketAlert.title,
});
});
for (request in groupedRERequests) {
const marketAlert = groupedRERequests[request];
let extractedData = toAWSArray(marketAlert.marketAlertArray);
const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
const region = getRegionName(marketAlert.requestObject.region);
const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
destinations.push({
Destination: {
ToAddresses: [
email
marketAlert.requestObject.email
]
},
ReplacementTemplateData: repData
@@ -147,18 +188,17 @@ const sendBulkEmail = async (marketAlerts) => {
} catch (e) {
console.log("Could not send bulk email", e)
}
}
const toAWSArray = (urlArray) => {
let arrayString = ""
urlArray.forEach(element => {
const formatetdTitle = element.title.replace(/"/g, "");
arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
});
return arrayString.slice(0, -1);
}
const getNotificationEmailHtml = () => {
@@ -179,8 +219,8 @@ const getNotificationEmailText = () => {
const createMarketAlertEmailTemplate = async () => {
const marketAlertTemplate = {
Template: {
TemplateName: "MarketAlertTemplate",
SubjectPart: "Javi mi obavijest",
TemplateName: TEMPLATE_NAME,
SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}",
TextPart: getNotificationEmailText(),
HtmlPart: getNotificationEmailHtml()
}
@@ -195,6 +235,10 @@ const createMarketAlertEmailTemplate = async () => {
}
}
const getSubject = (realEstateType, region, municipality) => {
return `${getRealEstateTypeEnum(realEstateType).title} ${getRegionName(region)}, ${getMunicipalityName(region, municipality)}`
}
module.exports = {
sendTemplatedEmail,
sendBulkEmail,

9
app/helpers/crawlers/olxClawler.js
View File

@@ -13,7 +13,7 @@ module.exports = class OlxCrawler {
this.maxResults = maxResults;
}
async indexSingle(url, email) {
async indexSingle(url, email, uuid) {
try {
const res = await fetch(url);
const body = await res.text();
@@ -81,6 +81,7 @@ module.exports = class OlxCrawler {
const parsedPrice = parsePrice(price);
console.log(location);
const locationArray = location.split(",");
const region = locationArray[0];
const municipality = locationArray[1];
@@ -88,6 +89,7 @@ module.exports = class OlxCrawler {
const data = {
realEstateType: this.getCategoryId(realEstateType),
email: email,
uuid: uuid,
olxId: olxId,
// category: category,
url,
@@ -138,7 +140,7 @@ module.exports = class OlxCrawler {
for (let i = 0; i < hrefs.length; i++) {
console.log(`indexing: ${hrefs[i]}`);
const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid);
if (singleData) {
results.push(singleData);
@@ -223,7 +225,8 @@ module.exports = class OlxCrawler {
const olxUrl = {
url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
email: request.email
email: request.email,
uuid: request.uniqueId
}
console.log(olxUrl.url);
urls.push(olxUrl);

15
app/helpers/db/dbHelper.js
View File

@@ -11,6 +11,20 @@ const allRERequest = async () => {
});
}
/**
* Find all subscribed RealEstateRequests by UUID
*/
const allRERequestByUiid = async (requestArray) => {
const Op = db.Sequelize.Op;
return await db.RealEstateRequest.findAll({
where: {
subscribed: true,
[Op.or]: requestArray
}
});
}
/**
* Find all , or all depending on notified bolean marketalerts, and order them by email
*
@@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => {
module.exports = {
allRERequest,
allMarketAlerts,
allRERequestByUiid,
findPointInsideBoundingBox
};

20
app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js Normal file
View File

@@ -0,0 +1,20 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
'MarketAlerts',
'request',
{
type: Sequelize.STRING
}
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn(
'MarketAlerts',
'request'
);
}
};

1
app/models/marketalert.js
View File

@@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => {
realEstateType : DataTypes.STRING,
notified : DataTypes.BOOLEAN,
title : DataTypes.STRING,
request: DataTypes.STRING,
email: {
type: DataTypes.STRING,

1
app/services/crawlerService.js
View File

@@ -33,6 +33,7 @@ async function crawlAll() {
size: result.size,
price: result.price,
email: result.email,
request: result.uuid,
// lastDate: DataTypes.STRING,
municipality: result.municipality,
region: result.region,

3
app/services/notificationService.js
View File

@@ -16,7 +16,7 @@ async function processNotifications() {
await sendBulkEmail(marketAlerts);
} else {
console.log("NOTIFICATION SERVICE: No new alerts");
return;
process.exit();
}
await db.MarketAlert.update(
@@ -26,6 +26,7 @@ async function processNotifications() {
process.exit();
} catch (e) {
console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
process.exit();
}
}