Merge branch 'notification-email-subject' into 'master'

Notification email subject

See merge request saburly/marketalarm/web!19

README.md

@@ -34,3 +34,10 @@ this will create and run postgres image and then execute migrations
 
 5. Run app  
 `npm start` or `npm run start-mon` to run app with automatic restart on code change
+
+
+### AWS SES
+
+    - AWS SES credentials are handled with env vratiables
+    - Notification emails are sent in batches of 50, by using SES templates
+    - Make sure that you are using different templates for different envirorments

app/helpers/awsEmail.js

@@ -2,8 +2,10 @@
 const dotenv = require('dotenv').config();
 const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
+const db = require('../models/index');
+const { allRERequestByUiid } = require('./db/dbHelper');
 var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
+const TEMPLATE_NAME = "MarketAlertTemplateDevelop"
 
 AWS.config.update({
   region: process.env.AMAZON_REGION,
@@ -37,7 +39,7 @@ const sendTemplatedEmail = async (email, request) => {
       },
       Subject: {
         Charset: 'UTF-8',
-        Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}`
+        Data: `Javimi Potvrda: ${getSubject(request.realEstateType, request.region, request.municipality)}`
       }
     },
     Source: process.env.SOURCE_EMAIL, /* required */
@@ -97,26 +99,65 @@ const sendBulkEmail = async (marketAlerts) => {
   try {
 
     destinations = []
-    groupedEmails = [];
+    groupedRERequests = [];
 
-    marketAlerts.forEach(marketAlert => {
+
-      if (!groupedEmails[marketAlert.email]) {
+    let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request);
-        groupedEmails[marketAlert.email] = [];
+
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+    RERequestUuids = Array.from(new Set(RERequestUuids));
-      } else {
+
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+    RERequestUuids = RERequestUuids.map(marketAlert => {
-      }
+      return { uniqueId: marketAlert }
     });
 
-    for (email in groupedEmails) {
+    const RERequest = await allRERequestByUiid(RERequestUuids);
+    const requestDataValues = [];
 
-      const url = groupedEmails[email];
+    RERequest.forEach(RERequest => {
-      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
+      var formatedRequest = {};
+      formatedRequest[RERequest.uniqueId] =
+        requestDataValues[RERequest.uniqueId] = {
+          realEstateType: RERequest.realEstateType,
+          region: RERequest.region,
+          municipality: RERequest.municipality
+        };
+    });
+
+    marketAlerts.forEach(marketAlert => {
+
+      const requestObject = {
+        email: marketAlert.email,
+        realEstateType: requestDataValues[marketAlert.request].realEstateType,
+        municipality: requestDataValues[marketAlert.request].municipality,
+        region: requestDataValues[marketAlert.request].region,
+      }
+
+      if (!groupedRERequests[marketAlert.request]) {
+        groupedRERequests[marketAlert.request] = {
+          requestObject: requestObject,
+          marketAlertArray: []
+        };
+      }
+
+      groupedRERequests[marketAlert.request].marketAlertArray.push({
+        url: marketAlert.url,
+        title: marketAlert.title,
+      });
+    });
+    for (request in groupedRERequests) {
+
+      const marketAlert = groupedRERequests[request];
+      let extractedData = toAWSArray(marketAlert.marketAlertArray);
+      const realEstateType = getRealEstateTypeEnum(marketAlert.requestObject.realEstateType).title;
+      const region = getRegionName(marketAlert.requestObject.region);
+      const municipality = getMunicipalityName(marketAlert.requestObject.region, marketAlert.requestObject.municipality);
+
+      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}" }`
 
       destinations.push({
         Destination: {
           ToAddresses: [
-            email
+            marketAlert.requestObject.email
           ]
         },
         ReplacementTemplateData: repData
@@ -147,18 +188,17 @@ const sendBulkEmail = async (marketAlerts) => {
   } catch (e) {
     console.log("Could not send bulk email", e)
   }
-
-
 }
 
 const toAWSArray = (urlArray) => {
+
   let arrayString = ""
   urlArray.forEach(element => {
     const formatetdTitle = element.title.replace(/"/g, "");
     arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
   });
-
   return arrayString.slice(0, -1);
+
 }
 
 const getNotificationEmailHtml = () => {
@@ -179,8 +219,8 @@ const getNotificationEmailText = () => {
 const createMarketAlertEmailTemplate = async () => {
   const marketAlertTemplate = {
     Template: {
-      TemplateName: "MarketAlertTemplate",
+      TemplateName: TEMPLATE_NAME,
-      SubjectPart: "Javi mi obavijest",
+      SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}",
       TextPart: getNotificationEmailText(),
       HtmlPart: getNotificationEmailHtml()
     }
@@ -195,6 +235,10 @@ const createMarketAlertEmailTemplate = async () => {
   }
 }
 
+const getSubject = (realEstateType, region, municipality) => {
+  return `${getRealEstateTypeEnum(realEstateType).title} ${getRegionName(region)}, ${getMunicipalityName(region, municipality)}`
+}
+
 module.exports = {
   sendTemplatedEmail,
   sendBulkEmail,

app/helpers/crawlers/olxClawler.js

@@ -13,7 +13,7 @@ module.exports = class OlxCrawler {
         this.maxResults = maxResults;
     }
 
-    async indexSingle(url, email) {
+    async indexSingle(url, email, uuid) {
         try {
             const res = await fetch(url);
             const body = await res.text();
@@ -81,6 +81,7 @@ module.exports = class OlxCrawler {
 
             const parsedPrice = parsePrice(price);
 
+            console.log(location);
             const locationArray = location.split(",");
             const region = locationArray[0];
             const municipality = locationArray[1];
@@ -88,6 +89,7 @@ module.exports = class OlxCrawler {
             const data = {
                 realEstateType: this.getCategoryId(realEstateType),
                 email: email,
+                uuid: uuid,
                 olxId: olxId,
                 // category: category,
                 url,
@@ -138,7 +140,7 @@ module.exports = class OlxCrawler {
             for (let i = 0; i < hrefs.length; i++) {
                 console.log(`indexing: ${hrefs[i]}`);
 
-                const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
+                const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid);
 
                 if (singleData) {
                     results.push(singleData);
@@ -223,7 +225,8 @@ module.exports = class OlxCrawler {
 
             const olxUrl = {
                 url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
-                email: request.email
+                email: request.email,
+                uuid: request.uniqueId
             }
             console.log(olxUrl.url);
             urls.push(olxUrl);

app/helpers/db/dbHelper.js

@@ -11,6 +11,20 @@ const allRERequest = async () => {
       });
 }
 
+/**
+ * Find all subscribed RealEstateRequests by UUID
+ */
+const allRERequestByUiid = async (requestArray) => {
+
+  const Op = db.Sequelize.Op;
+  return await db.RealEstateRequest.findAll({
+      where: {
+          subscribed: true,
+          [Op.or]: requestArray
+      }
+    });
+}
+
 /**
  * Find all , or all depending on notified bolean marketalerts, and order them by email
  * 
@@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => {
 module.exports = {
   allRERequest,
   allMarketAlerts,
+  allRERequestByUiid,
   findPointInsideBoundingBox
 };

app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'request',
+        {
+          type: Sequelize.STRING
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'request'
+    );
+  }
+};

app/models/marketalert.js

@@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => {
     realEstateType : DataTypes.STRING,
     notified : DataTypes.BOOLEAN,
     title : DataTypes.STRING,
+    request: DataTypes.STRING,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -33,6 +33,7 @@ async function crawlAll() {
                     size: result.size,
                     price: result.price,
                     email: result.email,
+                    request: result.uuid,
                     // lastDate: DataTypes.STRING,
                     municipality: result.municipality,
                     region: result.region,

app/services/notificationService.js

@@ -16,7 +16,7 @@ async function processNotifications() {
             await sendBulkEmail(marketAlerts);
         } else {
             console.log("NOTIFICATION SERVICE: No new alerts");
-            return;
+            process.exit();
         }
 
         await db.MarketAlert.update(
@@ -26,6 +26,7 @@ async function processNotifications() {
         process.exit();
     } catch (e) {
         console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
+        process.exit();
     }
 }