From b79a274f96fb6eb223ae156c2e4f698149229795 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Tue, 2 Jul 2019 21:49:56 +0200 Subject: [PATCH] Added formated subject to bulk email --- README.md | 6 ++ app/helpers/awsEmail.js | 83 +++++++++++++++---- app/helpers/crawlers/olxClawler.js | 9 +- app/helpers/db/dbHelper.js | 15 ++++ ...132143-add-RREquet-uuid-to-marketalerts.js | 20 +++++ app/models/marketalert.js | 1 + app/services/crawlerService.js | 1 + app/services/notificationService.js | 3 +- 8 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js diff --git a/README.md b/README.md index 6f2d0f4..c6609b0 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,9 @@ this will create and run postgres image and then execute migrations 5. Run app `npm start` or `npm run start-mon` to run app with automatic restart on code change + + +### AWS SES + + - AWS SES credentials are handled with env vratiables + - Notification emails are sent in batches of 50, by using SES templates diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index e9ffe9d..4eef45e 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -2,8 +2,10 @@ const dotenv = require('dotenv').config(); const { getRealEstateTypeEnum } = require('./enums'); const { getRegionName, getMunicipalityName } = require('./codes'); +const db = require('../models/index'); +const { allRERequestByUiid } = require('./db/dbHelper'); var AWS = require('aws-sdk'); -const TEMPLATE_NAME = "MarketAlertTemplate" +const TEMPLATE_NAME = "MarketAlertTemplateDevelop" AWS.config.update({ region: process.env.AMAZON_REGION, @@ -97,26 +99,78 @@ const sendBulkEmail = async (marketAlerts) => { try { destinations = [] - groupedEmails = []; + groupedRERequests = []; + + + let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request); + // console.log(RERequestUuids) + RERequestUuids = Array.from(new Set(RERequestUuids)); + // console.log(RERequestUuids) + RERequestUuids = RERequestUuids.map(marketAlert => { + return { uniqueId: marketAlert } + }); + // RERequestUuids = Array.from(new Set(RERequestUuids)); + // console.log(RERequestUuids) + // const RERequests = await + const RERequest = await allRERequestByUiid(RERequestUuids); + // const requestDataValues = RERequest.map(RERequest => { + // var formatedRequest = {}; + // formatedRequest[RERequest.uniqueId] = { + // realEstateType: RERequest.realEstateType, + // region: RERequest.region, + // municipality: RERequest.municipality + // }; + // return formatedRequest; + // }) + const requestDataValues = []; + RERequest.forEach(RERequest => { + var formatedRequest = {}; + formatedRequest[RERequest.uniqueId] = + requestDataValues[RERequest.uniqueId] = { + realEstateType: RERequest.realEstateType, + region: RERequest.region, + municipality: RERequest.municipality + }; + }); + // console.log(requestDataValues); + marketAlerts.forEach(marketAlert => { - if (!groupedEmails[marketAlert.email]) { - groupedEmails[marketAlert.email] = []; - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); - } else { - groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + // console.log(marketAlert); + // const RERequest = await currentRERequest(marketAlert.request); + if (!groupedRERequests[marketAlert.request]) { + groupedRERequests[marketAlert.request] = []; } + // console.log(requestDataValues[marketAlert.request]); + console.log("MarketAlertEmail"); + console.log(marketAlert.email); + + groupedRERequests[marketAlert.request].push({ + + url: marketAlert.url, + title: marketAlert.title, + email: marketAlert.email, + requestData: { + realEstateType: requestDataValues[marketAlert.request].realEstateType, + region: requestDataValues[marketAlert.request].region, + municipality: requestDataValues[marketAlert.request].municipality + } + }); }); - for (email in groupedEmails) { + for (request in groupedRERequests) { + // {{realestateType}}, {{region}}, {{municipality}} - const url = groupedEmails[email]; - let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }` + const marketAlert = groupedRERequests[request]; + console.log("RequestlertEmail"); + console.log(request.email); + console.log(request); + let repData = `{ "marketAlertUrl":[${toAWSArray(marketAlert)}], "realestateType":"${request.realEstateType}", "region":"${request.region}", "municipality":"${request.municipality}" }` destinations.push({ Destination: { ToAddresses: [ - email + request.email ] }, ReplacementTemplateData: repData @@ -124,7 +178,7 @@ const sendBulkEmail = async (marketAlerts) => { } console.log("AWS EMAIL : Bulk email replacement data:"); - console.log(destinations); + // console.log(destinations); var params = { Destinations: @@ -152,6 +206,7 @@ const sendBulkEmail = async (marketAlerts) => { } const toAWSArray = (urlArray) => { + console.log(urlArray); let arrayString = "" urlArray.forEach(element => { const formatetdTitle = element.title.replace(/"/g, ""); @@ -179,8 +234,8 @@ const getNotificationEmailText = () => { const createMarketAlertEmailTemplate = async () => { const marketAlertTemplate = { Template: { - TemplateName: "MarketAlertTemplate", - SubjectPart: "Javi mi obavijest", + TemplateName: TEMPLATE_NAME, + SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}", TextPart: getNotificationEmailText(), HtmlPart: getNotificationEmailHtml() } diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 556d9f3..18d25bb 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -13,7 +13,7 @@ module.exports = class OlxCrawler { this.maxResults = maxResults; } - async indexSingle(url, email) { + async indexSingle(url, email, uuid) { try { const res = await fetch(url); const body = await res.text(); @@ -81,6 +81,7 @@ module.exports = class OlxCrawler { const parsedPrice = parsePrice(price); + console.log(location); const locationArray = location.split(","); const region = locationArray[0]; const municipality = locationArray[1]; @@ -88,6 +89,7 @@ module.exports = class OlxCrawler { const data = { realEstateType: this.getCategoryId(realEstateType), email: email, + uuid: uuid, olxId: olxId, // category: category, url, @@ -138,7 +140,7 @@ module.exports = class OlxCrawler { for (let i = 0; i < hrefs.length; i++) { console.log(`indexing: ${hrefs[i]}`); - const singleData = await this.indexSingle(hrefs[i], olxUrl.email); + const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid); if (singleData) { results.push(singleData); @@ -223,7 +225,8 @@ module.exports = class OlxCrawler { const olxUrl = { url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`, - email: request.email + email: request.email, + uuid: request.uniqueId } console.log(olxUrl.url); urls.push(olxUrl); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index 3363e1b..234aff5 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -11,6 +11,20 @@ const allRERequest = async () => { }); } +/** + * Find all subscribed RealEstateRequests by UUID + */ +const allRERequestByUiid = async (requestArray) => { + + const Op = db.Sequelize.Op; + return await db.RealEstateRequest.findAll({ + where: { + subscribed: true, + [Op.or]: requestArray + } + }); +} + /** * Find all , or all depending on notified bolean marketalerts, and order them by email * @@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => { module.exports = { allRERequest, allMarketAlerts, + allRERequestByUiid, findPointInsideBoundingBox }; diff --git a/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js new file mode 100644 index 0000000..b904e6f --- /dev/null +++ b/app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'request', + { + type: Sequelize.STRING + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'request' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index c23961b..72c797c 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => { realEstateType : DataTypes.STRING, notified : DataTypes.BOOLEAN, title : DataTypes.STRING, + request: DataTypes.STRING, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 57d667a..e7ac968 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -33,6 +33,7 @@ async function crawlAll() { size: result.size, price: result.price, email: result.email, + request: result.uuid, // lastDate: DataTypes.STRING, municipality: result.municipality, region: result.region, diff --git a/app/services/notificationService.js b/app/services/notificationService.js index f299330..00d8fbf 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -16,7 +16,7 @@ async function processNotifications() { await sendBulkEmail(marketAlerts); } else { console.log("NOTIFICATION SERVICE: No new alerts"); - return; + process.exit(); } await db.MarketAlert.update( @@ -26,6 +26,7 @@ async function processNotifications() { process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e); + process.exit(); } }