From 96e9da1fb1c811bf884cb669ee1662c804784f78 Mon Sep 17 00:00:00 2001 From: Nedim Uka Date: Fri, 28 Jun 2019 18:06:19 +0200 Subject: [PATCH] Send templated bulk email, and remember notifed marketalerts --- app/helpers/awsEmail.js | 93 +++++++++++-------- app/helpers/crawlers/olxClawler.js | 2 +- app/helpers/db/dbHelper.js | 20 ++-- ...0190628165512-add-title-to-marketalerts.js | 20 ++++ app/models/marketalert.js | 1 + app/services/crawlerService.js | 6 +- app/services/notificationService.js | 17 +++- 7 files changed, 98 insertions(+), 61 deletions(-) create mode 100644 app/migrations/20190628165512-add-title-to-marketalerts.js diff --git a/app/helpers/awsEmail.js b/app/helpers/awsEmail.js index 0519c16..f72a896 100644 --- a/app/helpers/awsEmail.js +++ b/app/helpers/awsEmail.js @@ -28,11 +28,11 @@ const sendTemplatedEmail = async (email, request) => { Body: { /* required */ Html: { Charset: "UTF-8", - Data: getEmailHTML(request) + Data: getGreetingsEmailHTML(request) }, Text: { Charset: "UTF-8", - Data: getEmaiTextVersion(request) + Data: getGreetingsEmaiTextVersion(request) } }, Subject: { @@ -50,7 +50,7 @@ const sendTemplatedEmail = async (email, request) => { await sendEmailPromise; } -const getEmailHTML = (realestateRequest) => { +const getGreetingsEmailHTML = (realestateRequest) => { const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType); const gardenSize = realEstateType.hasGardenSize ? `
Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2
` : `` @@ -76,7 +76,7 @@ Javimi tim. } -const getEmaiTextVersion = (realestateRequest) => { +const getGreetingsEmaiTextVersion = (realestateRequest) => { const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType); const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : "" @@ -95,45 +95,39 @@ const getEmaiTextVersion = (realestateRequest) => { const sendBulkEmail = async (marketAlerts) => { try { - // Create the promise and SES service object - // const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).getTemplate({ TemplateName: TEMPLATE_NAME }).promise(); - // const template = await templatePromise; - // console.log(template); destinations = [] + groupedEmails = []; marketAlerts.forEach(marketAlert => { + if (!groupedEmails[marketAlert.email]) { + groupedEmails[marketAlert.email] = []; + groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + } else { + groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title }); + } + }); + + for (email in groupedEmails) { + + const url = groupedEmails[email]; + let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }` + destinations.push({ Destination: { ToAddresses: [ - marketAlert.email + email ] }, - ReplacementTemplateData: `{ "marketAlertUrl":"${marketAlert.url}", "favoriteanimal":"yak" }` + ReplacementTemplateData: repData }) - }); + } + console.log(destinations); - - // Create sendBulkTemplatedEmail params var params = { - Destinations: /* required */ - // { - // Destination: { /* required */ - // CcAddresses: [ - // 'EMAIL_ADDRESS', - // /* more items */ - // ], - // ToAddresses: [ - // 'EMAIL_ADDRESS', - // 'EMAIL_ADDRESS' - // /* more items */ - // ] - // }, - // ReplacementTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }' - // }, - // ], - destinations, + Destinations: + destinations, Source: process.env.SOURCE_EMAIL, /* required */ Template: TEMPLATE_NAME, /* required */ DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }', @@ -142,12 +136,8 @@ const sendBulkEmail = async (marketAlerts) => { ] }; - - - // Create the promise and SES service object - const sendPromise =new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); - + const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise(); await sendPromise; @@ -155,20 +145,45 @@ const sendBulkEmail = async (marketAlerts) => { console.log("Could not send bulk email", e) } + +} + +const toAWSArray = (urlArray) => { + let arrayString = "" + urlArray.forEach(element => { + arrayString = arrayString + `{"url":"${element.url}" , "title":"${element.title}"},` + }); + + return arrayString.slice(0, -1);; +} + +const getNotificationEmailHtml = () => { + return `

Zdravo, + Pronašli smo nekretninu koju ste tražili.

+

Ovo su tražene nekretnine:

+
+
{{#each marketAlertUrl}}
  • {{title}}

    • {{/each}}
    +
    +
    ` +} + +const getNotificationEmailText = () => { + return ` Zdravo, + Pronašli smo nekretninu koju ste tražili. Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}` } const createMarketAlertEmailTemplate = async () => { const marketAlertTemplate = { Template: { TemplateName: "MarketAlertTemplate", - SubjectPart: "Greetings", + SubjectPart: "Javi mi obavijest", TextPart: "Dear ,\r\nYour favorite animal is {{marketAlertUrl}}.", - HtmlPart: "

    Hello

    Your favorite animal is {{marketAlertUrl}}.

    " + HtmlPart: getNotificationEmailHtml() } } try { - const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).createTemplate(marketAlertTemplate).promise(); + const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise(); await templatePromise } catch (e) { @@ -176,8 +191,6 @@ const createMarketAlertEmailTemplate = async () => { } } - - module.exports = { sendTemplatedEmail, sendBulkEmail, diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js index 4c0a0a0..556d9f3 100644 --- a/app/helpers/crawlers/olxClawler.js +++ b/app/helpers/crawlers/olxClawler.js @@ -27,7 +27,7 @@ module.exports = class OlxCrawler { // } //TODO remove properties that are not needed, and add some if they are missing - const title = $('#naslovartikla').text(); + const title = $('#naslovartikla').text().trim(); const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text(); const price = $('#pc > p:nth-child(2)').text(); diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js index f81961d..3363e1b 100644 --- a/app/helpers/db/dbHelper.js +++ b/app/helpers/db/dbHelper.js @@ -11,15 +11,15 @@ const allRERequest = async () => { }); } - /** - * Find all unnotified marketalerts, and order them by email + * Find all , or all depending on notified bolean marketalerts, and order them by email * + * @param fechAll bolean * @param notified bolean * * @returns array of MarketAlerts */ -const allMarketAlerts = async (notified) => { +const allMarketAlerts = async (fetchAll, notified) => { let queryObject = { order: [ @@ -27,27 +27,19 @@ const allMarketAlerts = async (notified) => { ] } - if (notified){ + if (!fetchAll){ queryObject.where = { notified: notified } } - return await db.MarketAlert.findAll(queryObject); - // return await db.MarketAlerts.findAll({ - // where: { - // notified: notified - // }, - // order: [ - // ['email', 'DESC'], - // ] - // }); + return await db.MarketAlert.findAll(queryObject); } /** * Find all unnotified marketalerts * @param latLng array - * @param email strig + * @param email string * * @returns array of MarketAlerts */ diff --git a/app/migrations/20190628165512-add-title-to-marketalerts.js b/app/migrations/20190628165512-add-title-to-marketalerts.js new file mode 100644 index 0000000..d874f3d --- /dev/null +++ b/app/migrations/20190628165512-add-title-to-marketalerts.js @@ -0,0 +1,20 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.addColumn( + 'MarketAlerts', + 'title', + { + type: Sequelize.STRING + } + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.removeColumn( + 'MarketAlerts', + 'title' + ); + } +}; diff --git a/app/models/marketalert.js b/app/models/marketalert.js index 6f58560..c23961b 100644 --- a/app/models/marketalert.js +++ b/app/models/marketalert.js @@ -12,6 +12,7 @@ module.exports = (sequelize, DataTypes) => { region : DataTypes.STRING, realEstateType : DataTypes.STRING, notified : DataTypes.BOOLEAN, + title : DataTypes.STRING, email: { type: DataTypes.STRING, diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 756e9bf..57d667a 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -19,7 +19,7 @@ async function crawlAll() { try { - const marketAlertsFromDb = await allMarketAlerts(); + const marketAlertsFromDb = await allMarketAlerts(true); console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length); const marketAlerts = []; @@ -37,7 +37,9 @@ async function crawlAll() { municipality: result.municipality, region: result.region, gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize, - realEstateType: result.realEstateType + realEstateType: result.realEstateType, + title: result.title, + notified: false }) } console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length); diff --git a/app/services/notificationService.js b/app/services/notificationService.js index fb342f4..14d45d4 100644 --- a/app/services/notificationService.js +++ b/app/services/notificationService.js @@ -8,11 +8,20 @@ const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/aw async function processNotifications() { try { - const marketAlerts = await allMarketAlerts(false); - // await createMarketAlertEmailTemplate(); - await sendBulkEmail(marketAlerts); - // console.log(marketAlerts); + const marketAlerts = await allMarketAlerts(false, false); + console.log(marketAlerts.length) + await createMarketAlertEmailTemplate(); + if (marketAlerts.length > 0) { + console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length) + await sendBulkEmail(marketAlerts); + } else { + console.log("NOTIFICATION SERVICE: No new alerts"); + } + await db.MarketAlert.update( + { notified: true }, /* set attributes' value */ + { where: { notified: false } } /* where criteria */ + ); process.exit(); } catch (e) { console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);