From b79a274f96fb6eb223ae156c2e4f698149229795 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Tue, 2 Jul 2019 21:49:56 +0200 Subject: [PATCH 1/2] Added formated subject to bulk email --- README.md | 6 ++ app/helpers/awsEmail.js | 83 +++++++++++++++---- app/helpers/crawlers/olxClawler.js | 9 +- app/helpers/db/dbHelper.js | 15 ++++ ...132143-add-RREquet-uuid-to-marketalerts.js | 20 +++++ app/models/marketalert.js | 1 + app/services/crawlerService.js | 1 + app/services/notificationService.js | 3 +- 8 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js diff --git a/README.md b/README.md index 6f2d0f4..c6609b0 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,9 @@ this will create and run postgres image and then execute migrations 5. Run app `npm start` or `npm run start-mon` to run app with automatic restart on code change + + +### AWS SES + + - AWS SES credentials are handled with env vratiables + - Notification emails are sent in batches of 50, by using SES templates diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index e9ffe9d..4eef45e 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -2,8 +2,10 @@ const dotenv = require('dotenv').config(); const { getRealEstateTypeEnum } = require('./enums'); const { getRegionName, getMunicipalityName } = require('./codes'); +const db = require('../models/index'); +const { allRERequestByUiid } = require('./db/dbHelper'); var AWS = require('aws-sdk'); -const TEMPLATE_NAME = "MarketAlertTemplate" +const TEMPLATE_NAME = "MarketAlertTemplateDevelop" AWS.config.update({ region: process.env.AMAZON_REGION, @@ -97,26 +99,78 @@ const sendBulkEmail = async (marketAlerts) => { try { destinations = [] - groupedEmails = []; + groupedRERequests = []; + + + let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request); + // console.log(RERequestUuids) + RERequestUuids = Array.from(new Set(RERequestUuids)); + // console.log(RERequestUuids) + RERequestUuids = RERequestUuids.map(marketAlert => { + return { uniqueId: marketAlert } + }); + // RERequestUuids = Array.from(new Set(RERequestUuids)); + // console.log(RERequestUuids) + // const RERequests = await + const RERequest = await allRERequestByUiid(RERequestUuids); + // const requestDataValues = RERequest.map(RERequest => { + // var formatedRequest = {}; + // formatedRequest[RERequest.uniqueId] = { + // realEstateType: RERequest.realEstateType, + // region: RERequest.region, + // municipality: RERequest.municipality + // }; + // return formatedRequest; + // }) + const requestDataValues = []; + RERequest.forEach(RERequest => { + var formatedRequest = {}; + formatedRequest[RERequest.uniqueId] = + requestDataValues[RERequest.uniqueId] = { + realEstateType: RERequest.realEstateType, + region: RERequest.region, + municipality: RERequest.municipality + }; + }); + // console.log(requestDataValues); + marketAlerts.forEach(marketAlert => { - if (!groupedEmails[marketAlert.email]) { - groupedEmails[marketAlert.email] = []; - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); - } else { - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + // console.log(marketAlert); + // const RERequest = await currentRERequest(marketAlert.request); + if (!groupedRERequests[marketAlert.request]) { + groupedRERequests[marketAlert.request] = []; } + // console.log(requestDataValues[marketAlert.request]); + console.log("MarketAlertEmail"); + console.log(marketAlert.email); + + groupedRERequests[marketAlert.request].push({ + + url: marketAlert.url, + title: marketAlert.title, + email: marketAlert.email, + requestData: { + realEstateType: requestDataValues[marketAlert.request].realEstateType, + region: requestDataValues[marketAlert.request].region, + municipality: requestDataValues[marketAlert.request].municipality + } + }); }); - for (email in groupedEmails) { + for (request in groupedRERequests) { + // {{realestateType}}, {{region}}, {{municipality}} - const url = groupedEmails[email]; - let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }` + const marketAlert = groupedRERequests[request]; + console.log("RequestlertEmail"); + console.log(request.email); + console.log(request); + let repData = `{ "marketAlertUrl":[${toAWSArray(marketAlert)}], "realestateType":"${request.realEstateType}", "region":"${request.region}", "municipality":"${request.municipality}" }` destinations.push({ Destination: { ToAddresses: [ - email + request.email ] }, ReplacementTemplateData: repData @@ -124,7 +178,7 @@ const sendBulkEmail = async (marketAlerts) => { } console.log("AWS EMAIL : Bulk email replacement data:"); - console.log(destinations); + // console.log(destinations); var params = { Destinations: @@ -152,6 +206,7 @@ const sendBulkEmail = async (marketAlerts) => { } const toAWSArray = (urlArray) => { + console.log(urlArray); let arrayString = "" urlArray.forEach(element => { const formatetdTitle = element.title.replace(/"/g, ""); @@ -179,8 +234,8 @@ const getNotificationEmailText = () => { const createMarketAlertEmailTemplate = async () => { const marketAlertTemplate = { Template: { - TemplateName: "MarketAlertTemplate", - SubjectPart: "Javi mi obavijest", + TemplateName: TEMPLATE_NAME, + SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}", TextPart: getNotificationEmailText(), HtmlPart: getNotificationEmailHtml() } diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 556d9f3..18d25bb 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -13,7 +13,7 @@ module.exports = class OlxCrawler { this.maxResults = maxResults; } - async indexSingle(url, email) { + async indexSingle(url, email, uuid) { try { const res = await fetch(url); const body = await res.text(); @@ -81,6 +81,7 @@ module.exports = class OlxCrawler { const parsedPrice = parsePrice(price); + console.log(location); const locationArray = location.split(","); const region = locationArray[0]; const municipality = locationArray[1]; @@ -88,6 +89,7 @@ module.exports = class OlxCrawler { const data = { realEstateType: this.getCategoryId(realEstateType), email: email, + uuid: uuid, olxId: olxId, // category: category, url, @@ -138,7 +140,7 @@ module.exports = class OlxCrawler { for (let i = 0; i < hrefs.length; i++) { console.log(`indexing: ${hrefs[i]}`); - const singleData = await this.indexSingle(hrefs[i], olxUrl.email); + const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid); if (singleData) { results.push(singleData); @@ -223,7 +225,8 @@ module.exports = class OlxCrawler { const olxUrl = { url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`, - email: request.email + email: request.email, + uuid: request.uniqueId } console.log(olxUrl.url); urls.push(olxUrl); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index 3363e1b..234aff5 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -11,6 +11,20 @@ const allRERequest = async () => { }); } +/** + * Find all subscribed RealEstateRequests by UUID + */ +const allRERequestByUiid = async (requestArray) => { + + const Op = db.Sequelize.Op; + return await db.RealEstateRequest.findAll({ + where: { + subscribed: true, + [Op.or]: requestArray + } + }); +} + /** * Find all , or all depending on notified bolean marketalerts, and order them by email * @@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => { module.exports = { allRERequest, allMarketAlerts, + allRERequestByUiid, findPointInsideBoundingBox }; diff --git a/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js new file mode 100644 index 0000000..b904e6f --- /dev/null +++ b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'request', + { + type: Sequelize.STRING + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'request' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index c23961b..72c797c 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => { realEstateType : DataTypes.STRING, notified : DataTypes.BOOLEAN, title : DataTypes.STRING, + request: DataTypes.STRING, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 57d667a..e7ac968 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -33,6 +33,7 @@ async function crawlAll() { size: result.size, price: result.price, email: result.email, + request: result.uuid, // lastDate: DataTypes.STRING, municipality: result.municipality, region: result.region, diff --git a/app/services/notificationService.js b/app/services/notificationService.js index f299330..00d8fbf 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -16,7 +16,7 @@ async function processNotifications() { await sendBulkEmail(marketAlerts); } else { console.log("NOTIFICATION SERVICE: No new alerts"); - return; + process.exit(); } await db.MarketAlert.update( @@ -26,6 +26,7 @@ async function processNotifications() { process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); + process.exit(); } } From a807cb5bf23cdc0c5edee9330c995a64cf832aca Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Wed, 3 Jul 2019 15:13:59 +0200 Subject: [PATCH 2/2] Bulk emali subject --- README.md | 1 + app/helpers/awsEmail.js | 103 ++++++++++++++++++---------------------- 2 files changed, 47 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index c6609b0..2488e3c 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,4 @@ this will create and run postgres image and then execute migrations - AWS SES credentials are handled with env vratiables - Notification emails are sent in batches of 50, by using SES templates + - Make sure that you are using different templates for different envirorments diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index 4eef45e..ae590c0 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -39,7 +39,7 @@ const sendTemplatedEmail = async (email, request) => { }, Subject: { Charset: 'UTF-8', - Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}` + Data: `Javimi Potvrda: ${getSubject(request.realEstateType, request.region, request.municipality)}` } }, Source: process.env.SOURCE_EMAIL, /* required */ @@ -103,74 +103,61 @@ const sendBulkEmail = async (marketAlerts) => { let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request); - // console.log(RERequestUuids) + RERequestUuids = Array.from(new Set(RERequestUuids)); - // console.log(RERequestUuids) + RERequestUuids = RERequestUuids.map(marketAlert => { return { uniqueId: marketAlert } }); - // RERequestUuids = Array.from(new Set(RERequestUuids)); - // console.log(RERequestUuids) - // const RERequests = await + const RERequest = await allRERequestByUiid(RERequestUuids); - // const requestDataValues = RERequest.map(RERequest => { - // var formatedRequest = {}; - // formatedRequest[RERequest.uniqueId] = { - // realEstateType: RERequest.realEstateType, - // region: RERequest.region, - // municipality: RERequest.municipality - // }; - // return formatedRequest; - // }) const requestDataValues = []; + RERequest.forEach(RERequest => { var formatedRequest = {}; - formatedRequest[RERequest.uniqueId] = - requestDataValues[RERequest.uniqueId] = { - realEstateType: RERequest.realEstateType, - region: RERequest.region, - municipality: RERequest.municipality - }; - }); - // console.log(requestDataValues); - - - marketAlerts.forEach(marketAlert => { - // console.log(marketAlert); - // const RERequest = await currentRERequest(marketAlert.request); - if (!groupedRERequests[marketAlert.request]) { - groupedRERequests[marketAlert.request] = []; - } - // console.log(requestDataValues[marketAlert.request]); - console.log("MarketAlertEmail"); - console.log(marketAlert.email); - - groupedRERequests[marketAlert.request].push({ - - url: marketAlert.url, - title: marketAlert.title, - email: marketAlert.email, - requestData: { - realEstateType: requestDataValues[marketAlert.request].realEstateType, - region: requestDataValues[marketAlert.request].region, - municipality: requestDataValues[marketAlert.request].municipality - } - }); + formatedRequest[RERequest.uniqueId] = + requestDataValues[RERequest.uniqueId] = { + realEstateType: RERequest.realEstateType, + region: RERequest.region, + municipality: RERequest.municipality + }; }); + marketAlerts.forEach(marketAlert => { + + const requestObject = { + email: marketAlert.email, + realEstateType: requestDataValues[marketAlert.request].realEstateType, + municipality: requestDataValues[marketAlert.request].municipality, + region: requestDataValues[marketAlert.request].region, + } + + if (!groupedRERequests[marketAlert.request]) { + groupedRERequests[marketAlert.request] = { + requestObject: requestObject, + marketAlertArray: [] + }; + } + + groupedRERequests[marketAlert.request].marketAlertArray.push({ + url: marketAlert.url, + title: marketAlert.title, + }); + }); for (request in groupedRERequests) { - // {{realestateType}}, {{region}}, {{municipality}} const marketAlert = groupedRERequests[request]; - console.log("RequestlertEmail"); - console.log(request.email); - console.log(request); - let repData = `{ "marketAlertUrl":[${toAWSArray(marketAlert)}], "realestateType":"${request.realEstateType}", "region":"${request.region}", "municipality":"${request.municipality}" }` + let extractedData = toAWSArray(marketAlert.marketAlertArray); + const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title; + const region = getRegionName(marketAlert.requestObject.region); + const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality); + + let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }` destinations.push({ Destination: { ToAddresses: [ - request.email + marketAlert.requestObject.email ] }, ReplacementTemplateData: repData @@ -178,7 +165,7 @@ const sendBulkEmail = async (marketAlerts) => { } console.log("AWS EMAIL : Bulk email replacement data:"); - // console.log(destinations); + console.log(destinations); var params = { Destinations: @@ -201,19 +188,17 @@ const sendBulkEmail = async (marketAlerts) => { } catch (e) { console.log("Could not send bulk email", e) } - - } const toAWSArray = (urlArray) => { - console.log(urlArray); + let arrayString = "" urlArray.forEach(element => { const formatetdTitle = element.title.replace(/"/g, ""); arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},` }); - return arrayString.slice(0, -1); + } const getNotificationEmailHtml = () => { @@ -250,6 +235,10 @@ const createMarketAlertEmailTemplate = async () => { } } +const getSubject = (realEstateType, region, municipality) => { + return `${getRealEstateTypeEnum(realEstateType).title} ${getRegionName(region)}, ${getMunicipalityName(region, municipality)}` +} + module.exports = { sendTemplatedEmail, sendBulkEmail,