Formated title

Crawler service

See merge request saburly/marketalarm/web!17

app/helpers/awsEmail.js

@@ -1,9 +1,10 @@
 
-const dotenv = require('dotenv');
-dotenv.config();
+const dotenv = require('dotenv').config();
 const { getRealEstateTypeEnum } = require('./enums');
 const { getRegionName, getMunicipalityName } = require('./codes');
-const AWS = require('aws-sdk');
+var AWS = require('aws-sdk');
+const TEMPLATE_NAME = "MarketAlertTemplate"
+
 AWS.config.update({
   region: process.env.AMAZON_REGION,
   credentials:
@@ -27,11 +28,11 @@ const sendTemplatedEmail = async (email, request) => {
       Body: { /* required */
         Html: {
           Charset: "UTF-8",
-          Data: getEmailHTML(request)
+          Data: getGreetingsEmailHTML(request)
         },
         Text: {
           Charset: "UTF-8",
-          Data: getEmaiTextVersion(request)
+          Data: getGreetingsEmaiTextVersion(request)
         }
       },
       Subject: {
@@ -49,7 +50,7 @@ const sendTemplatedEmail = async (email, request) => {
   await sendEmailPromise;
 }
 
-const getEmailHTML = (realestateRequest) => {
+const getGreetingsEmailHTML = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
   const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``
 
@@ -75,15 +76,15 @@ Javimi tim.
 
 }
 
-const getEmaiTextVersion = (realestateRequest) => {
+const getGreetingsEmaiTextVersion = (realestateRequest) => {
   const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
-  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax  : ""
+  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
 
   const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
     + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
     + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
     + gardenSize
-    "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
+  "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
     "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
     "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
     "\n Tvoj,\n Javimi tim"
@@ -91,6 +92,111 @@ const getEmaiTextVersion = (realestateRequest) => {
   return text;
 }
 
+const sendBulkEmail = async (marketAlerts) => {
+
+  try {
+
+    destinations = []
+    groupedEmails = [];
+
+    marketAlerts.forEach(marketAlert => {
+      if (!groupedEmails[marketAlert.email]) {
+        groupedEmails[marketAlert.email] = [];
+        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+      } else {
+        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
+      }
+    });
+
+    for (email in groupedEmails) {
+
+      const url = groupedEmails[email];
+      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
+
+      destinations.push({
+        Destination: {
+          ToAddresses: [
+            email
+          ]
+        },
+        ReplacementTemplateData: repData
+      })
+
+    }
+    console.log("AWS EMAIL : Bulk email replacement data:");
+    console.log(destinations);
+
+    var params = {
+      Destinations:
+        destinations,
+      Source: process.env.SOURCE_EMAIL, /* required */
+      Template: TEMPLATE_NAME, /* required */
+      DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
+      ReplyToAddresses: [
+        process.env.SOURCE_EMAIL,
+      ]
+    };
+
+    // Create the promise and SES service object
+    const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
+    const awsResult = await sendPromise;
+    console.log("AWS SES bulk email response");
+    console.log(awsResult);
+
+
+  } catch (e) {
+    console.log("Could not send bulk email", e)
+  }
+
+
+}
+
+const toAWSArray = (urlArray) => {
+  let arrayString = ""
+  urlArray.forEach(element => {
+    const formatetdTitle = element.title.replace(/"/g, "");
+    arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
+  });
+
+  return arrayString.slice(0, -1);
+}
+
+const getNotificationEmailHtml = () => {
+  return `<h2> Zdravo, 
+  Pronašli smo nekretninu koju ste tražili. </h2>
+  <h3> Ovo su tražene nekretnine:  </h3> 
+  <div>
+  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
+  <div/>
+  </div>`
+}
+
+const getNotificationEmailText = () => {
+  return ` Zdravo, 
+  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
+}
+
+const createMarketAlertEmailTemplate = async () => {
+  const marketAlertTemplate = {
+    Template: {
+      TemplateName: "MarketAlertTemplate",
+      SubjectPart: "Javi mi obavijest",
+      TextPart: getNotificationEmailText(),
+      HtmlPart: getNotificationEmailHtml()
+    }
+  }
+
+  try {
+    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
+    await templatePromise
+
+  } catch (e) {
+    console.log("Could not create MarketAlertEmailTemplate", e);
+  }
+}
+
 module.exports = {
-  sendTemplatedEmail
+  sendTemplatedEmail,
+  sendBulkEmail,
+  createMarketAlertEmailTemplate
 };

app/helpers/crawlers/olxClawler.js

@@ -1,6 +1,6 @@
 const fetch = require('node-fetch');
 const cheerio = require('cheerio');
-const { allRERequest, findPointInsideBoundingBox } = require('../url');
+const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
 const { getRealEstateTypeEnum } = require('../enums');
 const { getRegion, getMunicipality } = require('../codes')
 const Promise = require("bluebird");
@@ -27,7 +27,7 @@ module.exports = class OlxCrawler {
             // }
 
             //TODO remove properties that are not needed, and add some if they are missing
-            const title = $('#naslovartikla').text();
+            const title = $('#naslovartikla').text().trim();
             const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
 
             const price = $('#pc > p:nth-child(2)').text();
@@ -87,14 +87,14 @@ module.exports = class OlxCrawler {
 
             const data = {
                 realEstateType: this.getCategoryId(realEstateType),
-                email : email,
+                email: email,
                 olxId: olxId,
                 // category: category,
                 url,
                 title,
-                price: isNaN(parsedPrice) ? price : parsedPrice,
+                price: isNaN(parsedPrice) ? 0 : parsedPrice,
                 size: parseFloat(size),
-                gardenSize: parseFloat(gardenSize),
+                gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
                 address,
                 region,
                 municipality,
@@ -118,6 +118,7 @@ module.exports = class OlxCrawler {
 
     async indexPage(olxUrl, maxResults = 1000) {
         try {
+            //TODO fix paging
             // console.log('Starting to index page: ' + pageNr);
             // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
 
@@ -142,7 +143,6 @@ module.exports = class OlxCrawler {
                 if (singleData) {
                     results.push(singleData);
                 }
-                // await this.sleep(500);
             }
 
             return results;
@@ -151,55 +151,61 @@ module.exports = class OlxCrawler {
         }
     }
 
-    getCategoryId (category) {
-        if (category === 'Stanovi') {
-          return 'stan';
-        } else if (category === 'Vikendice') {
-          return 'vikendica';
-        } else if (category === 'Kuće') {
-          return 'kuca';
+    getCategoryId(category) {
+
+        switch (category) {
+            case 'Stanovi':
+                return 'stan';
+
+            case 'Vikendice':
+                return 'vikendica'
+
+            case 'Kuće':
+                return 'kuca';
+
+            default:
+                return '';
         }
-      }
+    }
 
     async indexPages(urls, start, end, maxResults = 1000) {
+        //TODO fix paging
         // let results = {};
         // for (let i = start; i <= end; i++) {
         //     let result = await this.indexPage(i, maxResults);
         //     Object.assign(results, result)
-        //     await this.sleep(5000);
         // }
         // return results;
 
         let results = [];
         for (let url of urls) {
             let result = await this.indexPage(url, maxResults);
-            // Object.assign(results, result)
             results.push(result);
-            // await this.sleep(5000);
         }
         return results;
     }
 
     async crawl() {
+        console.log("OLX CRAWLER: start crawl");
 
         const filteredResults = [];
-        const realestateRequests = await allRERequest()
+        const realestateRequests = await allRERequest();
+        console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
         const urls = this.createRequestUrls(realestateRequests);
         let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
 
         for (const result of results) {
-            for (const re1 of result) {
-                if (re1.lat !== undefined && re1.lat !== null && re1.lat !== "") {
-                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
+            for (const finalResult of result) {
+                if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
 
                     if (pointInsideBoundingBox[0].length !== 0) {
-                        filteredResults.push(re1);
+                        filteredResults.push(finalResult);
                     }
                 }
             }
         }
-
-        console.log(filteredResults);
+        console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
         return filteredResults;
     }
 
@@ -216,7 +222,7 @@ module.exports = class OlxCrawler {
             const priceMax = "do=" + request.priceMax;
 
             const olxUrl = {
-                url: "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax,
+                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
                 email: request.email
             }
             console.log(olxUrl.url);

app/helpers/db/dbHelper.js

@@ -1,10 +1,54 @@
+const db = require('../../models/index');
 
-// const db = require('../../models/index');
+/**
+ * Find all subscribed RealEstateRequests 
+ */
+const allRERequest = async () => {
+    return await db.RealEstateRequest.findAll({
+        where: {
+            subscribed: true
+        }
+      });
+}
 
+/**
+ * Find all , or all depending on notified bolean marketalerts, and order them by email
+ * 
+ * @param fechAll bolean
+ * @param notified bolean
+ * 
+ * @returns array of MarketAlerts
+ */
+const allMarketAlerts = async (fetchAll, notified) => {
 
-// const bulkInsert = async (reuslts) => {
-//     db.MarketAlert.bulkCreate({
+    let queryObject = {
+        order: [
+            ['email', 'DESC'],
+        ]
+    }
 
-//     })
-   
-//   }
+    if (!fetchAll){
+        queryObject.where = {
+            notified: notified
+        }
+    }
+
+    return await db.MarketAlert.findAll(queryObject);
+  }
+  
+/**
+ * Find all unnotified marketalerts 
+ *  @param latLng array
+ *  @param email string
+ * 
+ * @returns array of MarketAlerts
+ */
+const findPointInsideBoundingBox = async (latLng, email) => {
+  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
+}
+
+module.exports = {
+  allRERequest,
+  allMarketAlerts,
+  findPointInsideBoundingBox
+};

app/helpers/url.js

@@ -7,17 +7,6 @@ const currentRERequest = async (req) => {
   const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
   return request;
 };
-// TODO Fetch only subscribed realestate requests
-const allRERequest = async () => {
-  return await db.RealEstateRequest.findAll();
-}
-
-const findPointInsideBoundingBox = async (latLng) => {
-  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
-}
-
 module.exports = {
   currentRERequest,
-  allRERequest,
-  findPointInsideBoundingBox
 };

app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'notified',
+        {
+          type: Sequelize.BOOLEAN
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'notified'
+    );
+  }
+};

app/migrations/20190628165512-add-title-to-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'title',
+        {
+          type: Sequelize.STRING
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'title'
+    );
+  }
+};

app/models/marketalert.js

@@ -11,6 +11,8 @@ module.exports = (sequelize, DataTypes) => {
     municipality : DataTypes.STRING,
     region : DataTypes.STRING,
     realEstateType : DataTypes.STRING,
+    notified : DataTypes.BOOLEAN,
+    title : DataTypes.STRING,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -2,6 +2,7 @@
 const Promise = require("bluebird");
 const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
+const { allMarketAlerts } = require('../helpers/db/dbHelper');
 
 const olxCrawler = new OlxCrawler(1, 2, 3);
 
@@ -10,6 +11,7 @@ const crawlers = [
 ];
 
 async function crawlAll() {
+    console.log("CRAWLER SERVICE: crawlAll");
 
     Promise.map(crawlers, function (crawler) {
         return crawler.crawl();
@@ -17,7 +19,8 @@ async function crawlAll() {
 
         try {
 
-            const marketAlertsFromDb = await db.MarketAlert.findAll();
+            const marketAlertsFromDb = await allMarketAlerts(true);
+            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
 
             const marketAlerts = [];
             const mergedResults = [].concat.apply([], results);
@@ -34,21 +37,30 @@ async function crawlAll() {
                     municipality: result.municipality,
                     region: result.region,
                     gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
-                    realEstateType: result.realEstateType
+                    realEstateType: result.realEstateType,
+                    title: result.title,
+                    notified: false
                 })
             }
+            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
+
             try {
-                const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+
+                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
+
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);
-                process.exit()
+                process.exit();
+
             } catch (e) {
-                console.log("Could not bulkCreate marketalers reason: ", e);
+                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
+                process.exit();
             }
         } catch (e) {
-            console.log("Error crawling. Trying next crawler! ", e);
+            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
+            process.exit();
         }
     })
 };
 
 crawlAll();
-

app/services/notificationService.js

@@ -0,0 +1,32 @@
+
+const Promise = require("bluebird");
+const db = require("../models/index");
+const { allMarketAlerts } = require('../helpers/db/dbHelper');
+const { createMarketAlertEmailTemplate, sendBulkEmail } = require('../helpers/awsEmail');
+
+
+async function processNotifications() {
+
+    try {
+        const marketAlerts = await allMarketAlerts(false, false);
+        console.log(marketAlerts.length)
+        await createMarketAlertEmailTemplate();
+        if (marketAlerts.length > 0) {
+            console.log("NOTIFICATION SERVICE: Number of new alerts: " + marketAlerts.length)
+            await sendBulkEmail(marketAlerts);
+        } else {
+            console.log("NOTIFICATION SERVICE: No new alerts");
+            return;
+        }
+
+        await db.MarketAlert.update(
+            { notified: true }, /* set attributes' value */
+            { where: { notified: false } } /* where criteria */
+        );
+        process.exit();
+    } catch (e) {
+        console.log("NOTIFICATION SERVICE: could not send notifications reason: ", e);
+    }
+}
+
+processNotifications();

package.json

@@ -7,7 +7,8 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
-    "scheduler": "node ./app/services/crawlerService.js",
+    "crawler": "node ./app/services/crawlerService.js",
+    "notification": "node ./app/services/notificationService.js",
     "migrate": "cd app && npx sequelize db:migrate",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",