Send templated bulk email, and remember notifed marketalerts

app/helpers/awsEmail.js

@@ -28,11 +28,11 @@ const sendTemplatedEmail = async (email, request) => {
       Body: { /* required */
         Html: {
           Charset: "UTF-8",
-          Data: getEmailHTML(request)
+          Data: getGreetingsEmailHTML(request)
         },
         Text: {
           Charset: "UTF-8",
-          Data: getEmaiTextVersion(request)
+          Data: getGreetingsEmaiTextVersion(request)
         }
       },
       Subject: {
@@ -50,7 +50,7 @@ const sendTemplatedEmail = async (email, request) => {
   await sendEmailPromise;
 }
 
-const getEmailHTML = (realestateRequest) => {
+const getGreetingsEmailHTML = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
   const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``
 
@@ -76,7 +76,7 @@ Javimi tim.
 
 }
 
-const getEmaiTextVersion = (realestateRequest) => {
+const getGreetingsEmaiTextVersion = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
   const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
 
@@ -95,45 +95,39 @@ const getEmaiTextVersion = (realestateRequest) => {
 const sendBulkEmail = async (marketAlerts) => {
 
   try {
-    // Create the promise and SES service object
-    // const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).getTemplate({ TemplateName: TEMPLATE_NAME }).promise();
-    // const template = await templatePromise;
-    // console.log(template);
 
     destinations = []
+    groupedEmails = [];
 
     marketAlerts.forEach(marketAlert => {
+      if (!groupedEmails[marketAlert.email]) {
+        groupedEmails[marketAlert.email] = [];
+        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+      } else {
+        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+      }
+    });
+
+    for (email in groupedEmails) {
+
+      const url = groupedEmails[email];
+      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
+
       destinations.push({
         Destination: {
           ToAddresses: [
-            marketAlert.email
+            email
           ]
         },
-        ReplacementTemplateData: `{ "marketAlertUrl":"${marketAlert.url}", "favoriteanimal":"yak" }`
+        ReplacementTemplateData: repData
       })
 
-    });
+    }
+    console.log(destinations);
 
-
-    // Create sendBulkTemplatedEmail params 
     var params = {
-      Destinations:  /* required */
+      Destinations:
-      //   {
+        destinations,
-      //     Destination: { /* required */
-      //       CcAddresses: [
-      //         'EMAIL_ADDRESS',
-      //         /* more items */
-      //       ],
-      //       ToAddresses: [
-      //         'EMAIL_ADDRESS',
-      //         'EMAIL_ADDRESS'
-      //         /* more items */
-      //       ]
-      //     },
-      //     ReplacementTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }'
-      //   },
-      // ],
-      destinations,
       Source: process.env.SOURCE_EMAIL, /* required */
       Template: TEMPLATE_NAME, /* required */
       DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
@@ -142,12 +136,8 @@ const sendBulkEmail = async (marketAlerts) => {
       ]
     };
 
-    
-  
-
     // Create the promise and SES service object
-    const sendPromise =new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
+    const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
-
     await sendPromise;
 
 
@@ -155,20 +145,45 @@ const sendBulkEmail = async (marketAlerts) => {
     console.log("Could not send bulk email", e)
   }
 
+
+}
+
+const toAWSArray = (urlArray) => {
+  let arrayString = ""
+  urlArray.forEach(element => {
+    arrayString = arrayString + `{"url":"${element.url}" , "title":"${element.title}"},`
+  });
+
+  return arrayString.slice(0, -1);;
+}
+
+const getNotificationEmailHtml = () => {
+  return `<h2> Zdravo, 
+  Pronašli smo nekretninu koju ste tražili. </h2>
+  <h3> Ovo su tražene nekretnine:  </h3> 
+  <div>
+  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
+  <div/>
+  </div>`
+}
+
+const getNotificationEmailText = () => {
+  return ` Zdravo, 
+  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
 }
 
 const createMarketAlertEmailTemplate = async () => {
   const marketAlertTemplate = {
     Template: {
       TemplateName: "MarketAlertTemplate",
-      SubjectPart: "Greetings",
+      SubjectPart: "Javi mi obavijest",
       TextPart: "Dear ,\r\nYour favorite animal is {{marketAlertUrl}}.",
-      HtmlPart: "<h1>Hello </h1><p>Your favorite animal is {{marketAlertUrl}}.</p>"
+      HtmlPart: getNotificationEmailHtml()
     }
   }
 
   try {
-    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).createTemplate(marketAlertTemplate).promise();
+    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
     await templatePromise
 
   } catch (e) {
@@ -176,8 +191,6 @@ const createMarketAlertEmailTemplate = async () => {
   }
 }
 
-
-
 module.exports = {
   sendTemplatedEmail,
   sendBulkEmail,

app/helpers/crawlers/olxClawler.js

@@ -27,7 +27,7 @@ module.exports = class OlxCrawler {
             // }
 
             //TODO remove properties that are not needed, and add some if they are missing
-            const title = $('#naslovartikla').text();
+            const title = $('#naslovartikla').text().trim();
             const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
 
             const price = $('#pc > p:nth-child(2)').text();

app/helpers/db/dbHelper.js

@@ -11,15 +11,15 @@ const allRERequest = async () => {
       });
 }
 
-
 /**
- * Find all unnotified marketalerts, and order them by email
+ * Find all , or all depending on notified bolean marketalerts, and order them by email
  * 
+ * @param fechAll bolean
  * @param notified bolean
  * 
  * @returns array of MarketAlerts
  */
-const allMarketAlerts = async (notified) => {
+const allMarketAlerts = async (fetchAll, notified) => {
 
     let queryObject = {
         order: [
@@ -27,27 +27,19 @@ const allMarketAlerts = async (notified) => {
         ]
     }
 
-    if (notified){
+    if (!fetchAll){
         queryObject.where = {
             notified: notified
         }
     }
-    return await db.MarketAlert.findAll(queryObject);
 
-    //     return await db.MarketAlerts.findAll({
+    return await db.MarketAlert.findAll(queryObject);
-    //     where: {
-    //         notified: notified
-    //     },
-    //     order: [
-    //         ['email', 'DESC'],
-    //     ]
-    //   });
   }
   
 /**
  * Find all unnotified marketalerts 
  *  @param latLng array
- *  @param email strig
+ *  @param email string
  * 
  * @returns array of MarketAlerts
  */

app/migrations/20190628165512-add-title-to-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'title',
+        {
+          type: Sequelize.STRING
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'title'
+    );
+  }
+};

app/models/marketalert.js

@@ -12,6 +12,7 @@ module.exports = (sequelize, DataTypes) => {
     region : DataTypes.STRING,
     realEstateType : DataTypes.STRING,
     notified : DataTypes.BOOLEAN,
+    title : DataTypes.STRING,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -19,7 +19,7 @@ async function crawlAll() {
 
         try {
 
-            const marketAlertsFromDb = await allMarketAlerts();
+            const marketAlertsFromDb = await allMarketAlerts(true);
             console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
 
             const marketAlerts = [];
@@ -37,7 +37,9 @@ async function crawlAll() {
                     municipality: result.municipality,
                     region: result.region,
                     gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType
+                    realEstateType: result.realEstateType,
+                    title: result.title,
+                    notified: false
                 })
             }
             console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);

app/services/notificationService.js

@@ -8,11 +8,20 @@ const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/aw
 async function processNotifications() {
 
     try {
-        const marketAlerts = await allMarketAlerts(false);
+        const marketAlerts = await allMarketAlerts(false, false);
-        // await createMarketAlertEmailTemplate();
+        console.log(marketAlerts.length)
-        await sendBulkEmail(marketAlerts);
+        await createMarketAlertEmailTemplate();
-        // console.log(marketAlerts);
+        if (marketAlerts.length > 0) {
+            console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
+            await sendBulkEmail(marketAlerts);
+        } else {
+            console.log("NOTIFICATION SERVICE: No new alerts");
+        }
 
+        await db.MarketAlert.update(
+            { notified: true }, /* set attributes' value */
+            { where: { notified: false } } /* where criteria */
+        );
         process.exit();
     } catch (e) {
         console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);