Added formated subject to bulk email

README.md

@@ -34,3 +34,9 @@ this will create and run postgres image and then execute migrations
 
 5. Run app  
 `npm start` or `npm run start-mon` to run app with automatic restart on code change
+
+
+### AWS SES
+
+    - AWS SES credentials are handled with env vratiables
+    - Notification emails are sent in batches of 50, by using SES templates

app/helpers/awsEmail.js

@@ -2,8 +2,10 @@
 const dotenv = require('dotenv').config();
 const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
+const db = require('../models/index');
+const { allRERequestByUiid } = require('./db/dbHelper');
 var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
+const TEMPLATE_NAME = "MarketAlertTemplateDevelop"
 
 AWS.config.update({
   region: process.env.AMAZON_REGION,
@@ -97,26 +99,78 @@ const sendBulkEmail = async (marketAlerts) => {
   try {
 
     destinations = []
-    groupedEmails = [];
+    groupedRERequests = [];
+
+
+    let RERequestUuids = marketAlerts.map(marketAlert => marketAlert.request);
+    // console.log(RERequestUuids)
+    RERequestUuids = Array.from(new Set(RERequestUuids));
+    // console.log(RERequestUuids)
+    RERequestUuids = RERequestUuids.map(marketAlert => {
+      return { uniqueId: marketAlert }
+    });
+    // RERequestUuids = Array.from(new Set(RERequestUuids));
+    // console.log(RERequestUuids)
+    // const RERequests = await 
+    const RERequest = await allRERequestByUiid(RERequestUuids);
+    // const requestDataValues = RERequest.map(RERequest => {
+    //   var formatedRequest = {};
+    //   formatedRequest[RERequest.uniqueId] = {
+    //     realEstateType: RERequest.realEstateType,
+    //     region: RERequest.region,
+    //     municipality: RERequest.municipality
+    //   };
+    //   return formatedRequest;
+    // })
+    const requestDataValues = [];
+    RERequest.forEach(RERequest => {
+      var formatedRequest = {};
+      formatedRequest[RERequest.uniqueId] = 
+      requestDataValues[RERequest.uniqueId] = {
+        realEstateType: RERequest.realEstateType,
+        region: RERequest.region,
+        municipality: RERequest.municipality
+      };
+   });
+    // console.log(requestDataValues);
+
 
     marketAlerts.forEach(marketAlert => {
-      if (!groupedEmails[marketAlert.email]) {
+      // console.log(marketAlert);
-        groupedEmails[marketAlert.email] = [];
+      // const RERequest = await currentRERequest(marketAlert.request);
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+      if (!groupedRERequests[marketAlert.request]) {
-      } else {
+        groupedRERequests[marketAlert.request] = [];
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
       }
+      // console.log(requestDataValues[marketAlert.request]);
+      console.log("MarketAlertEmail");
+      console.log(marketAlert.email);
+      
+      groupedRERequests[marketAlert.request].push({
+
+        url: marketAlert.url,
+        title: marketAlert.title,
+        email: marketAlert.email,
+        requestData: {
+          realEstateType: requestDataValues[marketAlert.request].realEstateType,
+          region: requestDataValues[marketAlert.request].region,
+          municipality: requestDataValues[marketAlert.request].municipality
+        }
+      });
     });
 
-    for (email in groupedEmails) {
+    for (request in groupedRERequests) {
+      // {{realestateType}}, {{region}}, {{municipality}}
 
-      const url = groupedEmails[email];
+      const marketAlert = groupedRERequests[request];
-      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
+      console.log("RequestlertEmail");
+      console.log(request.email);
+      console.log(request);
+      let repData = `{ "marketAlertUrl":[${toAWSArray(marketAlert)}], "realestateType":"${request.realEstateType}", "region":"${request.region}", "municipality":"${request.municipality}" }`
 
       destinations.push({
         Destination: {
           ToAddresses: [
-            email
+            request.email
           ]
         },
         ReplacementTemplateData: repData
@@ -124,7 +178,7 @@ const sendBulkEmail = async (marketAlerts) => {
 
     }
     console.log("AWS EMAIL : Bulk email replacement data:");
-    console.log(destinations);
+    // console.log(destinations);
 
     var params = {
       Destinations:
@@ -152,6 +206,7 @@ const sendBulkEmail = async (marketAlerts) => {
 }
 
 const toAWSArray = (urlArray) => {
+  console.log(urlArray);
   let arrayString = ""
   urlArray.forEach(element => {
     const formatetdTitle = element.title.replace(/"/g, "");
@@ -179,8 +234,8 @@ const getNotificationEmailText = () => {
 const createMarketAlertEmailTemplate = async () => {
   const marketAlertTemplate = {
     Template: {
-      TemplateName: "MarketAlertTemplate",
+      TemplateName: TEMPLATE_NAME,
-      SubjectPart: "Javi mi obavijest",
+      SubjectPart: "Javi mi obavijest: {{realestateType}}, {{region}}, {{municipality}}",
       TextPart: getNotificationEmailText(),
       HtmlPart: getNotificationEmailHtml()
     }

app/helpers/crawlers/olxClawler.js

@@ -13,7 +13,7 @@ module.exports = class OlxCrawler {
         this.maxResults = maxResults;
     }
 
-    async indexSingle(url, email) {
+    async indexSingle(url, email, uuid) {
         try {
             const res = await fetch(url);
             const body = await res.text();
@@ -81,6 +81,7 @@ module.exports = class OlxCrawler {
 
             const parsedPrice = parsePrice(price);
 
+            console.log(location);
             const locationArray = location.split(",");
             const region = locationArray[0];
             const municipality = locationArray[1];
@@ -88,6 +89,7 @@ module.exports = class OlxCrawler {
             const data = {
                 realEstateType: this.getCategoryId(realEstateType),
                 email: email,
+                uuid: uuid,
                 olxId: olxId,
                 // category: category,
                 url,
@@ -138,7 +140,7 @@ module.exports = class OlxCrawler {
             for (let i = 0; i < hrefs.length; i++) {
                 console.log(`indexing: ${hrefs[i]}`);
 
-                const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
+                const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid);
 
                 if (singleData) {
                     results.push(singleData);
@@ -223,7 +225,8 @@ module.exports = class OlxCrawler {
 
             const olxUrl = {
                 url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
-                email: request.email
+                email: request.email,
+                uuid: request.uniqueId
             }
             console.log(olxUrl.url);
             urls.push(olxUrl);

app/helpers/db/dbHelper.js

@@ -11,6 +11,20 @@ const allRERequest = async () => {
       });
 }
 
+/**
+ * Find all subscribed RealEstateRequests by UUID
+ */
+const allRERequestByUiid = async (requestArray) => {
+
+  const Op = db.Sequelize.Op;
+  return await db.RealEstateRequest.findAll({
+      where: {
+          subscribed: true,
+          [Op.or]: requestArray
+      }
+    });
+}
+
 /**
  * Find all , or all depending on notified bolean marketalerts, and order them by email
  * 
@@ -50,5 +64,6 @@ const findPointInsideBoundingBox = async (latLng, email) => {
 module.exports = {
   allRERequest,
   allMarketAlerts,
+  allRERequestByUiid,
   findPointInsideBoundingBox
 };

app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'request',
+        {
+          type: Sequelize.STRING
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'request'
+    );
+  }
+};

app/models/marketalert.js

@@ -13,6 +13,7 @@ module.exports = (sequelize, DataTypes) => {
     realEstateType : DataTypes.STRING,
     notified : DataTypes.BOOLEAN,
     title : DataTypes.STRING,
+    request: DataTypes.STRING,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -33,6 +33,7 @@ async function crawlAll() {
                     size: result.size,
                     price: result.price,
                     email: result.email,
+                    request: result.uuid,
                     // lastDate: DataTypes.STRING,
                     municipality: result.municipality,
                     region: result.region,

app/services/notificationService.js

@@ -16,7 +16,7 @@ async function processNotifications() {
             await sendBulkEmail(marketAlerts);
         } else {
             console.log("NOTIFICATION SERVICE: No new alerts");
-            return;
+            process.exit();
         }
 
         await db.MarketAlert.update(
@@ -26,6 +26,7 @@ async function processNotifications() {
         process.exit();
     } catch (e) {
         console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
+        process.exit();
     }
 }