Merge branch 'notification-service' into 'master'

Notification service

See merge request saburly/marketalarm/web!35

app/common/enums.js

@@ -4,12 +4,13 @@ const AD_TYPE = {
 };
 
 const AD_CATEGORY = {
-  CATEGORY_FLAT: "FLAT",
-  CATEGORY_HOUSE: "HOUSE",
-  CATEGORY_OFFICE: "OFFICE",
-  CATEGORY_LAND: "LAND",
-  CATEGORY_APARTMENT: "APARTMENT",
-  CATEGORY_GARAGE: "GARAGE"
+  FLAT: { id: "FLAT", title: "Stan", hasGardenSize: false },
+  HOUSE: { id: "HOUSE", title: "Kuća", hasGardenSize: true },
+  //OFFICE: { id: "OFFICE", title: "Kancelarija", hasGardenSize: false },
+  //LAND: { id: "LAND", title: "Zemljište", hasGardenSize: true },
+  APARTMENT: { id: "APARTMENT", title: "Apartman", hasGardenSize: false }
+  //GARAGE: { id: "GARAGE", title: "Garaža", hasGardenSize: false },
+  //COTTAGE: { id: "COTTAGE", title: "Vikendica", hasGardenSize: true }
 };
 
 const AD_STATUS = {

app/config/appConfig.js

@@ -1,3 +1,6 @@
+"use strict";
+require("dotenv").config({ path: __dirname + "/./../../.env" });
+
 const APP_PORT = process.env.PORT || 5000;
 const APP_BASE_URL = process.env.APP_BASE_URL || "http://localhost";
 
@@ -11,10 +14,24 @@ const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
 const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
 
+const AWS_EMAIL_CONFIG = {
+  REGION: process.env.AWS_REGION || "",
+  CREDENTIALS: {
+    ACCESS_KEY_ID: process.env.AWS_KEY_ID || "",
+    SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ""
+  },
+  SOURCE_EMAIL: process.env.SOURCE_EMAIL || ""
+};
+
+const MAX_REAL_ESTATES_IN_EMAIL =
+  parseInt(process.env.MAX_REAL_ESTATES_IN_EMAIL) || 10;
+
 module.exports = {
   APP_PORT,
   APP_URL,
   DEFAULT_TIMEZONE,
   CRAWLER_INTERVAL,
-  STOP_CRAWLER
+  STOP_CRAWLER,
+  AWS_EMAIL_CONFIG,
+  MAX_REAL_ESTATES_IN_EMAIL
 };

app/controllers/gardenSizes.js

@@ -1,5 +1,5 @@
 const { currentSearchRequest } = require("../helpers/url");
-const { getRealEstateTypeEnum } = require("../helpers/enums");
+const { AD_CATEGORY } = require("../common/enums");
 
 const getGardenSize = (req, res) => {
   const title = "Koliko okućnice tražite ?";
@@ -28,7 +28,7 @@ const postGardenSize = async (req, res) => {
   const nextStepPage = req.query.nextStep || "cijena";
   const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
 
-  const realEstateType = getRealEstateTypeEnum(searchRequest.realEstateType);
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
   if (realEstateType && realEstateType.hasGardenSize) {
     const gardenSizeMin = req.body.from || 0;
     const gardenSizeMax = req.body.to || 0;

app/controllers/queryReview.js

@@ -1,9 +1,5 @@
 const { currentSearchRequest } = require("../helpers/url");
-const {
-  realEstateTypes,
-  getEnumTypeTitle,
-  getRealEstateTypeEnum
-} = require("../helpers/enums");
+const { AD_CATEGORY } = require("../common/enums");
 
 const getQueryReview = async (req, res) => {
   const title = "Da li je ovo to što ste tražili ?";
@@ -25,13 +21,13 @@ const getQueryReview = async (req, res) => {
     priceMax
   } = searchRequest.dataValues;
 
-  const realEstateTypeObject = getRealEstateTypeEnum(realEstateType);
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
   const enableGardenSizeEdit = realEstateTypeObject
     ? realEstateTypeObject.hasGardenSize
     : false;
 
-  const realEstateTypeTitle = realEstateType
-    ? getEnumTypeTitle(realEstateTypes, realEstateType)
+  const realEstateTypeTitle = realEstateTypeObject
+    ? realEstateTypeObject.title
     : "-";
 
   const locationTitle = "Location description - PLACEHOLDER";

app/controllers/querySubmit.js

@@ -1,6 +1,8 @@
 const { currentSearchRequest } = require("../helpers/url");
 const { isValidEmail } = require("../helpers/email");
-const { sendTemplatedEmail } = require("../helpers/awsEmail");
+const {
+  notifyForNewSearchRequest
+} = require("../services/notificationService");
 
 const getQuerySubmit = async (req, res) => {
   const title = "Upišite vaš e-mail";
@@ -42,7 +44,8 @@ const postQuerySubmit = async (req, res) => {
   searchRequest.subscribed = true;
   await searchRequest.save();
 
-  sendTemplatedEmail(emailInput, searchRequest);
+  await notifyForNewSearchRequest(searchRequest);
+
   res.redirect(nextStep);
 };
 

app/controllers/realEstateTypes.js

@@ -1,10 +1,13 @@
 const { currentSearchRequest } = require("../helpers/url");
 const { createSearchRequest } = require("../helpers/db/searchRequest");
 
-const { realEstateTypes, getRealEstateTypeEnum } = require("../helpers/enums");
+const { AD_CATEGORY } = require("../common/enums");
 
 const getRealEstateTypes = (req, res) => {
   const title = "Koju nekretninu tražite?";
+  const realEstateTypes = Object.keys(AD_CATEGORY).map(
+    category => AD_CATEGORY[category]
+  );
   res.render("realEstateType", { realEstateTypes, title });
 };
 

app/controllers/realEstates.js

@@ -1,10 +1,13 @@
-const { allMarketAlertsByRequest } = require("../helpers/db/dbHelper");
+"use strict";
+const {
+  findRealEstatesForSearchRequest
+} = require("../helpers/db/searchRequestMatch");
 
 const getRealEstates = async (req, res) => {
-  const request = req.params["request_id"];
-  const realEstates = await allMarketAlertsByRequest(request);
+  const searchRequestId = req.params["searchRequestId"] || "";
+  const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
 
-  const title = "Ovo su nekretnine koje smo pronašli za vas";
+  const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
   res.render("realEstates", { realEstates, title });
 };
 

app/controllers/redirect.js

@@ -1,8 +1,8 @@
-const { getMarketAlertById } = require("../helpers/db/dbHelper");
+const { getRealEstateById } = require("../helpers/db/realEstate");
 
 const redirect = async (req, res) => {
   const id = req.params["id"];
-  const marketAlert = await getMarketAlertById(id);
+  const marketAlert = await getRealEstateById(id);
   if (marketAlert) {
     res.redirect(marketAlert.url);
   } else {

app/controllers/sizes.js

@@ -1,5 +1,5 @@
 const { currentSearchRequest } = require("../helpers/url");
-const { sizes, getRealEstateTypeEnum } = require("../helpers/enums");
+const { AD_CATEGORY } = require("../common/enums");
 
 const getSize = (req, res) => {
   const title = "Od koliko kvadrata tražite nekretninu ?";
@@ -24,7 +24,7 @@ const getSize = (req, res) => {
 const postSize = async (req, res) => {
   const searchRequest = await currentSearchRequest(req);
 
-  const realEstateType = getRealEstateTypeEnum(searchRequest.realEstateType);
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
   const sizeMin = req.body.from || 0;
   const sizeMax = req.body.to || 0;
   //TODO: Validation, check if real estate type is valid, ...

app/crawler/crawl.js

@@ -5,8 +5,6 @@
    All environment specific configuration is read here and
    passed to the crawlers and savers.
 */
-
-require("dotenv").config();
 const OlxCrawler = require("./specific/olx");
 const { OLX_CONFIG } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");

app/crawler/crawlerConfig.js

@@ -1,5 +1,5 @@
 "use strict";
-require("dotenv").config({ path: "../../.env" });
+require("dotenv").config({ path: __dirname + "/./../../.env" });
 const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
 
 const olxCrawlerAdType =
@@ -12,7 +12,7 @@ const olxParsedCrawlerAdCategories =
     ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
         category.trim()
       )
-    : ["CATEGORY_FLAT", "CATEGORY_HOUSE"];
+    : ["FLAT", "HOUSE"];
 
 const olxIgnoredUsernames =
   process.env.OLX_IGNORED_USERNAMES !== undefined
@@ -22,7 +22,9 @@ const olxIgnoredUsernames =
     : [];
 
 const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
-  .map(categoryName => AD_CATEGORY[categoryName])
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
   .filter(category => !!category);
 
 const OLX_CONFIG = {

app/crawler/specific/olx.js

@@ -22,12 +22,12 @@ const OLX_ENUMS = {
     [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
   },
   OLX_AD_CATEGORY: {
-    [AD_CATEGORY.CATEGORY_FLAT]: "&kategorija=23",
-    [AD_CATEGORY.CATEGORY_HOUSE]: "&kategorija=24",
-    [AD_CATEGORY.CATEGORY_LAND]: "&kategorija=29",
-    [AD_CATEGORY.CATEGORY_OFFICE]: "&kategorija=25",
-    [AD_CATEGORY.CATEGORY_APARTMENT]: "&kategorija=27",
-    [AD_CATEGORY.CATEGORY_GARAGE]: "&kategorija=30"
+    [AD_CATEGORY.FLAT.id]: "&kategorija=23",
+    [AD_CATEGORY.HOUSE.id]: "&kategorija=24",
+    //[AD_CATEGORY.LAND.id]: "&kategorija=29",
+    //[AD_CATEGORY.OFFICE.id]: "&kategorija=25",
+    [AD_CATEGORY.APARTMENT.id]: "&kategorija=27"
+    //[AD_CATEGORY.CATEGORY_GARAGE.id]: "&kategorija=30"
   },
   MAX_DETAIL_FIELDS: 30,
   OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
@@ -38,10 +38,7 @@ class OlxCrawler {
   constructor(
     savers = [],
     crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
-    crawlerAdCategories = [
-      AD_CATEGORY.CATEGORY_FLAT,
-      AD_CATEGORY.CATEGORY_HOUSE
-    ],
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
     maxPages = 1000,
     maxResultsPerPage = 100,
     ignoredUsernames = [],
@@ -196,7 +193,7 @@ class OlxCrawler {
   }
 
   async scrapeAd(url) {
-    //console.log("Scraping : ", url);
+    // console.log("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
@@ -407,7 +404,7 @@ class OlxCrawler {
         url,
         agencyObjectId: olxId,
         originAgencyName: AD_AGENCY.OLX,
-        realEstateType: this.getAdCategoryId(category),
+        realEstateType: parsedCategory,
         adType: parsedAdType,
         title,
         price: parsedPrice,
@@ -448,15 +445,15 @@ class OlxCrawler {
   getAdCategoryId(categoryText) {
     switch (categoryText) {
       case "Stanovi":
-        return AD_CATEGORY.CATEGORY_FLAT;
+        return AD_CATEGORY.FLAT.id;
       case "Zemljišta":
-        return AD_CATEGORY.CATEGORY_LAND;
+        return undefined; //AD_CATEGORY.LAND;
       case "Kuće":
-        return AD_CATEGORY.CATEGORY_HOUSE;
+        return AD_CATEGORY.HOUSE.id;
       case "Poslovni prostori":
-        return AD_CATEGORY.CATEGORY_OFFICE;
+        return undefined; //AD_CATEGORY.OFFICE;
       case "Apartmani":
-        return AD_CATEGORY.CATEGORY_APARTMENT;
+        return AD_CATEGORY.APARTMENT.id;
       default:
         return undefined;
     }

app/helpers/awsEmail.js

@@ -1,250 +0,0 @@
-const { APP_URL } = require("../config/appConfig");
-const { getRealEstateTypeEnum } = require("./enums");
-const { getRegionName, getMunicipalityName } = require("./codes");
-const { allRERequestByUiid } = require("./db/dbHelper");
-let AWS = require("aws-sdk");
-const TEMPLATE_NAME = "MarketAlertTemplate";
-
-AWS.config.update({
-  region: process.env.AMAZON_REGION,
-  credentials: {
-    accessKeyId: process.env.AMAZON_ACCES_KEY_ID,
-    secretAccessKey: process.env.AMAZON_SECRET_ACCESS_KEY
-  }
-});
-
-const sendTemplatedEmail = async (email, request) => {
-  const params = {
-    Destination: {
-      /* required */
-      CcAddresses: [],
-      ToAddresses: [email]
-    },
-    Message: {
-      /* required */
-      Body: {
-        /* required */
-        Html: {
-          Charset: "UTF-8",
-          Data: getGreetingsEmailHTML(request)
-        },
-        Text: {
-          Charset: "UTF-8",
-          Data: getGreetingsEmailTextVersion(request)
-        }
-      },
-      Subject: {
-        Charset: "UTF-8",
-        Data: `Javimi Potvrda: ${getSubject(request.realEstateType)}`
-      }
-    },
-    Source: process.env.SOURCE_EMAIL /* required */,
-    ReplyToAddresses: [process.env.SOURCE_EMAIL]
-  };
-
-  const sendEmailPromise = new AWS.SES({ apiVersion: "2010-12-01" })
-    .sendEmail(params)
-    .promise();
-  await sendEmailPromise;
-};
-
-const getGreetingsEmailHTML = realEstateRequest => {
-  const realEstateType = getRealEstateTypeEnum(
-    realEstateRequest.realEstateType
-  );
-  const gardenSize = realEstateType.hasGardenSize
-    ? `<div><strong>Kvadratura okućnice: Od ${realEstateRequest.gardenSizeMin} do ${realEstateRequest.gardenSizeMax} m2  </strong></div>`
-    : ``;
-
-  return `<h1> Zdravo, 
-  Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima. </h1>
-  <h2> Ovo je tražena nekretnina:  </h2> 
-  <div>
-  <div> <strong>Tip nekretnine: ${realEstateType.title} </strong></div>
-  <div><strong>Lokacija:  </strong></div>
-  <div><strong>Kvadratura nekretnine: Od ${realEstateRequest.sizeMin} do ${realEstateRequest.sizeMax} m2  </strong></div>
-  ${gardenSize}
-  <div><strong>Cijena: ${realEstateRequest.priceMin} do ${realEstateRequest.priceMax} KM  </strong></div>
-  </div>
-  <div> 
-
-  </div>
-  <div><strong> Ako želis prestati dobijati obavještenja za ovu pretragu klikni ${APP_URL}/odjava/${realEstateRequest.id} </strong></div>
-  <div><strong>Ako želiš promijeniti uslove pretrage klikni  ${APP_URL}/pregled/${realEstateRequest.id} </strong></div>
-    <h4> Tvoj,
-    Javimi tim.  
-    </h4>`;
-};
-
-const getGreetingsEmailTextVersion = realEstateRequest => {
-  const realEstateType = getRealEstateTypeEnum(
-    realEstateRequest.realEstateType
-  );
-  const gardenSize = realEstateType.hasGardenSize
-    ? `Kvadratura okućnice od ${realEstateRequest.gardenSizeMin} do ${realEstateRequest.gardenSizeMax}`
-    : "";
-
-  return `Zdravo\nNaručio/la si da ti javimo ako se nekretnina pojavi u oglasima\n
-  Ovo je tražena nekretnina:\nTip nekretnine: ${realEstateRequest.realEstateType}\n
-  Lokacija nekretnine :\n
-  Kvadratura nekretnine Od ${realEstateRequest.sizeMin} do ${realEstateRequest.sizeMax} 
-  ${gardenSize}\n
-  Cijena od ${realEstateRequest.priceMin} do ${realEstateRequest.priceMax} \n 
-  Ako želis prestati dobijati obavještenja za ovu pretragu klikni
-    ${APP_URL}/odjava/${realEstateRequest.id}\n 
-  Ako želiš promijeniti uslove pretrage klikni
-    ${APP_URL}/odpregled/${realEstateRequest.id}\n 
-   Tvoj,\n Javimi tim`;
-};
-
-const sendBulkEmail = async marketAlerts => {
-  try {
-    destinations = [];
-    groupedRERequests = [];
-
-    const RERequestUuidsMaped = marketAlerts.map(
-      marketAlert => marketAlert.request
-    );
-
-    const RERequestUuidsArray = Array.from(new Set(RERequestUuidsMaped));
-
-    const RERequestUuids = RERequestUuidsArray.map(marketAlert => {
-      return { id: marketAlert };
-    });
-
-    const RERequests = await allRERequestByUiid(RERequestUuids);
-    const requestDataValues = [];
-
-    RERequests.forEach(RERequest => {
-      var formatedRequest = {};
-      formatedRequest[RERequest.id] = requestDataValues[RERequest.id] = {
-        realEstateType: RERequest.realEstateType,
-        region: RERequest.region,
-        municipality: RERequest.municipality,
-        requestUrl: `${APP_URL}/nekretnine/${RERequest.id}`
-      };
-    });
-
-    marketAlerts.forEach(marketAlert => {
-      const requestObject = {
-        email: marketAlert.email,
-        realEstateType: requestDataValues[marketAlert.request].realEstateType,
-        municipality: requestDataValues[marketAlert.request].municipality,
-        region: requestDataValues[marketAlert.request].region,
-        requestUrl: requestDataValues[marketAlert.request].requestUrl
-      };
-
-      if (!groupedRERequests[marketAlert.request]) {
-        groupedRERequests[marketAlert.request] = {
-          requestObject: requestObject,
-          marketAlertArray: []
-        };
-      }
-
-      if (groupedRERequests[marketAlert.request].marketAlertArray.length < 10) {
-        groupedRERequests[marketAlert.request].marketAlertArray.push({
-          url: marketAlert.url,
-          title: marketAlert.title,
-          id: marketAlert.id
-        });
-      }
-    });
-
-    for (request in groupedRERequests) {
-      const marketAlert = groupedRERequests[request];
-      let extractedData = toAWSArray(marketAlert.marketAlertArray);
-      const realEstateType = getRealEstateTypeEnum(
-        marketAlert.requestObject.realEstateType
-      ).title;
-      const region = getRegionName(marketAlert.requestObject.region);
-      const municipality = getMunicipalityName(
-        marketAlert.requestObject.region,
-        marketAlert.requestObject.municipality
-      );
-      const requestUrl = marketAlert.requestObject.requestUrl;
-
-      let repData = `{ "marketAlertUrl":[${extractedData}], "realestateType":"${realEstateType}", "region":"${region}", "municipality":"${municipality}", "requestUrl":"${requestUrl}" }`;
-
-      destinations.push({
-        Destination: {
-          ToAddresses: [marketAlert.requestObject.email]
-        },
-        ReplacementTemplateData: repData
-      });
-    }
-
-    var params = {
-      Destinations: destinations,
-      Source: process.env.SOURCE_EMAIL /* required */,
-      Template: TEMPLATE_NAME /* required */,
-      DefaultTemplateData: '{ "REPLACEMENT_TAG_NAME":"REPLACEMENT_VALUE" }',
-      ReplyToAddresses: [process.env.SOURCE_EMAIL]
-    };
-
-    // Create the promise and SES service object
-    const sendPromise = new AWS.SES({ apiVersion: "2010-12-01" })
-      .sendBulkTemplatedEmail(params)
-      .promise();
-    const awsResult = await sendPromise;
-  } catch (e) {
-    console.log("Could not send bulk email", e);
-  }
-};
-
-const redirectUrl = marketAlertId => `${APP_URL}/redirect/${marketAlertId}`;
-const toAWSArray = urlArray => {
-  let arrayString = "";
-  urlArray.forEach(element => {
-    const formatetdTitle = element.title.replace(/"/g, "");
-    arrayString =
-      arrayString +
-      `{"url":"${redirectUrl(element.id)}" , "title":"${formatetdTitle}"},`;
-  });
-  return arrayString.slice(0, -1);
-};
-
-const getNotificationEmailHtml = () => {
-  return `<h2> Zdravo,
-  Pronašli smo nekretninu koju ste tražili. </h2>
-  <h3> Ovo su tražene nekretnine:  </h3>
-  <div>
-  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/>
-  <div/>
-  <div>Kompletan spisak nekretnina možete pegledati ovdije: <a href="{{requestUrl}}">Nekretnine</a> <div>
-  </div>`;
-};
-
-const getNotificationEmailText = () => {
-  return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}} , Kompletan spisan nekretnina mozete pegledati ovdije: {{requestUrl}}`;
-};
-
-const createMarketAlertEmailTemplate = async () => {
-  const marketAlertTemplate = {
-    Template: {
-      TemplateName: TEMPLATE_NAME,
-      SubjectPart: "Javi mi obavijest: {{realestateType}}",
-      TextPart: getNotificationEmailText(),
-      HtmlPart: getNotificationEmailHtml()
-    }
-  };
-
-  try {
-    const templatePromise = new AWS.SES({ apiVersion: "2010-12-01" })
-      .updateTemplate(marketAlertTemplate)
-      .promise();
-    await templatePromise;
-  } catch (e) {
-    console.log("Could not create MarketAlertEmailTemplate", e);
-  }
-};
-
-const getSubject = realEstateType => {
-  return getRealEstateTypeEnum(realEstateType).title;
-};
-
-module.exports = {
-  sendTemplatedEmail,
-  sendBulkEmail,
-  createMarketAlertEmailTemplate
-};

app/helpers/db/dbHelper.js

@@ -1,106 +0,0 @@
-const db = require("../../models/index");
-
-/**
- * Find all subscribed RealEstateRequests
- */
-const allRERequest = async () => {
-  return await db.RealEstateRequest.findAll({
-    where: {
-      subscribed: true
-    }
-  });
-};
-
-/**
- * Find all subscribed RealEstateRequests by UUID
- */
-const allRERequestByUiid = async requestArray => {
-  const Op = db.Sequelize.Op;
-  return await db.RealEstateRequest.findAll({
-    where: {
-      subscribed: true,
-      [Op.or]: requestArray
-    }
-  });
-};
-
-/**
- * Find all , or all depending on notified bolean marketalerts, that the hasLocation is true, and order them by email
- *
- * @param fechAll bolean
- * @param notified bolean
- *
- * @returns array of MarketAlerts
- */
-const allMarketAlerts = async (fetchAll, notified) => {
-  let queryObject = {
-    order: [["email", "DESC"]]
-  };
-
-  if (!fetchAll) {
-    queryObject.where = {
-      notified: notified,
-      hasLocation: true
-    };
-  }
-
-  return await db.MarketAlert.findAll(queryObject);
-};
-
-/**
- * Find all , MarketAlerts depending on request
- *
- * @param request string
- *
- * @returns array of MarketAlerts
- */
-const allMarketAlertsByRequest = async request => {
-  let queryObject = {
-    where: {
-      request: request
-    }
-  };
-
-  return await db.MarketAlert.findAll(queryObject);
-};
-
-/**
- * Find MarketAlerts by id
- *
- * @param id number
- *
- * @returns single MarketAlert
- */
-const getMarketAlertById = async id => {
-  let queryObject = {
-    where: {
-      id: id
-    }
-  };
-
-  return await db.MarketAlert.findOne(queryObject);
-};
-
-/**
- * Find all unnotified marketalerts
- *  @param latLng array
- *  @param email string
- *
- * @returns array of MarketAlerts
- */
-const findPointInsideBoundingBox = async (latLng, email, uniqueId) => {
-  return await db.sequelize.query(
-    `SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND "uniqueId" = '${uniqueId}' AND subscribed = true AND ST_Contains("RealEstateRequests"."boundingBox", ST_GEOMFROMTEXT('POINT (${
-      latLng[0]
-    } ${latLng[1]})'))`
-  );
-};
-
-module.exports = {
-  allRERequest,
-  allMarketAlerts,
-  allRERequestByUiid,
-  findPointInsideBoundingBox,
-  allMarketAlertsByRequest,
-  getMarketAlertById
-};

app/helpers/db/realEstate.js

@@ -36,6 +36,11 @@ const bulkUpsertRealEstates = async realEstateData => {
   }
 };
 
-module.exports = {
-  bulkUpsertRealEstates
+const getRealEstateById = async id => {
+  return db.RealEstate.findByPk(id);
+};
+
+module.exports = {
+  bulkUpsertRealEstates,
+  getRealEstateById
 };

app/helpers/db/searchRequest.js

@@ -1,5 +1,7 @@
 "use strict";
 const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
 
 const getSearchRequest = async searchRequestId => {
   return await db.SearchRequest.findByPk(searchRequestId);
@@ -9,7 +11,60 @@ const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
 
+const findSearchRequestsForRealEstate = async realEstate => {
+  const {
+    price,
+    area,
+    adType,
+    realEstateType,
+    locationLat,
+    locationLong
+  } = realEstate;
+
+  if (!locationLat || !locationLong) {
+    return [];
+  }
+
+  const stGeometry = sequelize.fn(
+    "ST_GEOMFROMTEXT",
+    `POINT (${locationLong} ${locationLat})`,
+    4326
+  );
+  const areaToSearchColumn = sequelize.col("areaToSearch");
+  const contains = sequelize.fn("ST_Contains", areaToSearchColumn, stGeometry);
+
+  const geoSearchQueryPart = sequelize.where(contains, true);
+
+  const query = {
+    adType,
+    realEstateType,
+    subscribed: true,
+    [Op.and]: geoSearchQueryPart
+  };
+
+  if (price) {
+    query.priceMin = {
+      [Op.lte]: price
+    };
+    query.priceMax = {
+      [Op.gte]: price
+    };
+  }
+
+  if (area) {
+    query.sizeMin = {
+      [Op.lte]: area
+    };
+    query.sizeMax = {
+      [Op.gte]: area
+    };
+  }
+
+  return await db.SearchRequest.findAll({ where: query });
+};
+
 module.exports = {
   getSearchRequest,
-  createSearchRequest
+  createSearchRequest,
+  findSearchRequestsForRealEstate
 };

app/helpers/db/searchRequestMatch.js

@@ -0,0 +1,33 @@
+"use strict";
+const db = require("../../models/index");
+
+const findRealEstatesForSearchRequest = async searchRequestId => {
+  const query = {
+    searchRequestId
+  };
+
+  const include = [{ model: db.RealEstate, as: "realEstates" }];
+
+  const matches = await db.SearchRequestMatch.findAll({
+    where: query,
+    include
+  });
+
+  const matchingRealEstates = [];
+  for (const match of matches) {
+    matchingRealEstates.push(...match.realEstates);
+  }
+
+  return matchingRealEstates;
+};
+
+const addMatches = async matchingRecords => {
+  return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
+    ignoreDuplicates: true
+  });
+};
+
+module.exports = {
+  findRealEstatesForSearchRequest,
+  addMatches
+};

app/helpers/emailContentGenerator.js

@@ -0,0 +1,78 @@
+"use strict";
+
+const { MAX_REAL_ESTATES_IN_EMAIL, APP_URL } = require("../config/appConfig");
+const { AD_CATEGORY } = require("../common/enums");
+
+const generateEmailFooter = searchRequestId => {
+  return `<div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
+     <div>Ako želite pogledati ili promijeniti uslove za ovu pretragu, <a href="${APP_URL}/pregled/${searchRequestId}">pogledajte ovdje</a></div>
+     <br/> 
+     <strong>Vaš,<br/>Javimi tim</strong>`;
+};
+
+const generateNotificationEmail = (realEstates, searchRequestId) => {
+  const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
+  const realEstatesToShow = truncateList
+    ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
+    : realEstates;
+
+  const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
+
+  let realEstateLinks = "";
+  for (const realEstate of realEstatesToShow) {
+    const { id: realEstateId, title } = realEstate;
+
+    realEstateLinks += `<li><a href="${APP_URL}/redirect/${realEstateId}">${title}</a></li><br />`;
+  }
+
+  const moreRealEstates = `<div>Kompletan spisak nekretnina možete pegledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
+
+  const emailFooter = generateEmailFooter(searchRequestId);
+
+  return `<h3>Zdravo</h3>
+  <h4>Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi</h4>
+  <div>
+  ${realEstateLinks}
+  <div/>
+  ${moreRealEstates}
+  </div>
+  <br/>
+  ${emailFooter}`;
+};
+
+const generateNewSearchRequestEmail = searchRequest => {
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  const {
+    id,
+    gardenSizeMin,
+    gardenSizeMax,
+    sizeMin,
+    sizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest;
+
+  const gardenSize = realEstateType.hasGardenSize
+    ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2  </strong></div>`
+    : ``;
+
+  const emailFooter = generateEmailFooter(id);
+
+  return `<h3>Zdravo</h3>
+  <div>Naručili ste da Vam javimo ako se nekretnina sa navedenim uslovima pojavi u oglasima:</div>
+  <br/>
+  <div>
+    <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Lokacija: </strong></div>
+    <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
+    ${gardenSize}
+    <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
+  </div>
+  <br/>
+  ${emailFooter}`;
+};
+
+module.exports = {
+  generateNotificationEmail,
+  generateNewSearchRequestEmail
+};

app/helpers/enums.js

@@ -1,57 +0,0 @@
-const realEstateTypes = [
-  { title: "Kuća", id: "kuca", hasGardenSize: true, olxid: 24 },
-  { title: "Stan", id: "stan", hasGardenSize: false, olxid: 23 },
-  { title: "Vikendica", id: "vikendica", hasGardenSize: true, olxid: 26 }
-];
-
-const sizes = [
-  { title: "do 50 m2", id: "50m2" },
-  { title: "do 75 m2", id: "75m2" },
-  { title: "do 100 m2", id: "100m2" },
-  { title: "do 150 m2", id: "150m2" },
-  { title: "do 200 m2", id: "200m2" },
-  { title: "preko 200 m2", id: "moreThan200m2" }
-];
-
-const gardenSizes = [
-  { title: "do 100 m2", id: "100m2" },
-  { title: "do 500 m2", id: "500m2" },
-  { title: "do 1 dunum", id: "1000m2" },
-  { title: "do 2 dunuma", id: "2000m2" },
-  { title: "do 3 dunuma", id: "3000m2" },
-  { title: "preko 3 dunuma", id: "moreThan3000m2" }
-];
-
-const prices = [
-  { title: "do 50 000 KM", id: "50kKM" },
-  { title: "do 100 000 KM", id: "100kKM" },
-  { title: "do 150 000 KM", id: "150kKM" },
-  { title: "do 200 000 KM", id: "200kKM" },
-  { title: "do 250 000 KM", id: "250kKM" },
-  { title: "preko 250 000 KM", id: "moreThan250kKM" }
-];
-
-const getEnumObject = (enumType, enumId) => {
-  return enumType.find(enumValue => enumValue.id === enumId);
-};
-
-const getRealEstateTypeEnum = enumId => {
-  return getEnumObject(realEstateTypes, enumId) || null;
-};
-
-const getEnumTypeTitle = (enumType, enumId) => {
-  const enumObject = getEnumObject(enumType, enumId);
-  if (!enumObject) {
-    return null;
-  }
-  return enumObject.title;
-};
-
-module.exports = {
-  realEstateTypes,
-  sizes,
-  gardenSizes,
-  prices,
-  getRealEstateTypeEnum,
-  getEnumTypeTitle
-};

app/migrations/20190927121449-change-adType-in-searchRequests-table-to-SALE.js

@@ -0,0 +1,15 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'SALE' WHERE "adType" = 'sell';`
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'sell' WHERE "adType" = 'SALE';`
+    );
+  }
+};

app/migrations/20190927180101-change-change-realEstateType-in-searchRequests-table.js

@@ -0,0 +1,31 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'HOUSE' WHERE "realEstateType" = 'kuca';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'FLAT' WHERE "realEstateType" = 'stan';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'COTTAGE' WHERE "realEstateType" = 'vikendica';`
+      )
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'kuca' WHERE "realEstateType" = 'HOUSE';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'stan' WHERE "realEstateType" = 'FLAT';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'vikendica' WHERE "realEstateType" = 'COTTAGE';`
+      )
+    ]);
+  }
+};

app/models/realEstate.js

@@ -51,11 +51,5 @@ module.exports = (sequelize, DataTypes) => {
     renewedDate: DataTypes.DATE
   });
 
-  RealEstate.associate = models => {
-    RealEstate.belongsToMany(models.SearchRequestMatch, {
-      through: "SearchRequestMatch"
-    });
-  };
-
   return RealEstate;
 };

app/models/searchRequest.js

@@ -1,5 +1,7 @@
 "use strict";
 
+const { AD_TYPE } = require("../common/enums");
+
 module.exports = (sequelize, DataTypes) => {
   const SearchRequest = sequelize.define("SearchRequest", {
     id: {
@@ -24,7 +26,7 @@ module.exports = (sequelize, DataTypes) => {
     adType: {
       type: DataTypes.TEXT,
       allowNull: false,
-      defaultValue: "sell"
+      defaultValue: AD_TYPE.AD_TYPE_SALE
     },
     email: DataTypes.TEXT,
     locality: DataTypes.TEXT,
@@ -62,11 +64,5 @@ module.exports = (sequelize, DataTypes) => {
     }
   });
 
-  SearchRequest.associate = models => {
-    SearchRequest.belongsToMany(models.SearchRequestMatch, {
-      through: "SearchRequestMatch"
-    });
-  };
-
   return SearchRequest;
 };

app/models/searchRequestMatch.js

@@ -9,15 +9,6 @@ module.exports = (sequelize, DataTypes) => {
         autoIncrement: true,
         allowNull: false
       },
-      searchRequestId: {
-        type: DataTypes.UUID,
-        allowNull: false,
-        primaryKey: true,
-        references: {
-          model: "SearchRequest",
-          key: "id"
-        }
-      },
       realEstateId: {
         type: DataTypes.BIGINT,
         allowNull: false,
@@ -29,6 +20,15 @@ module.exports = (sequelize, DataTypes) => {
         onUpdate: "CASCADE",
         onDelete: "SET NULL"
       },
+      searchRequestId: {
+        type: DataTypes.UUID,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "SearchRequest",
+          key: "id"
+        }
+      },
       notified: {
         type: DataTypes.BOOLEAN,
         allowNull: false,
@@ -43,5 +43,12 @@ module.exports = (sequelize, DataTypes) => {
     }
   );
 
+  SearchRequestMatch.associate = models => {
+    SearchRequestMatch.hasMany(models.RealEstate, {
+      foreignKey: "id",
+      as: "realEstates"
+    });
+  };
+
   return SearchRequestMatch;
 };

app/services/emailService.js

@@ -0,0 +1,56 @@
+"use strict";
+
+let AWS = require("aws-sdk");
+const htmlToText = require("html-to-text");
+
+const { AWS_EMAIL_CONFIG } = require("../config/appConfig");
+
+AWS.config.update({
+  region: AWS_EMAIL_CONFIG.REGION,
+  credentials: {
+    accessKeyId: AWS_EMAIL_CONFIG.CREDENTIALS.ACCESS_KEY_ID,
+    secretAccessKey: AWS_EMAIL_CONFIG.CREDENTIALS.SECRET_ACCESS_KEY
+  }
+});
+
+const awsMailer = new AWS.SES({ apiVersion: "2010-12-01" });
+
+const sendEmail = async (to, subject, message, from) => {
+  const params = {
+    Destination: {
+      ToAddresses: [to]
+    },
+    Message: {
+      Subject: {
+        Charset: "UTF-8",
+        Data: subject
+      },
+      Body: {
+        Html: {
+          Charset: "UTF-8",
+          Data: message
+        },
+        Text: {
+          Charset: "UTF-8",
+          Data: htmlToText.fromString(message)
+        }
+      }
+    },
+    ReturnPath: from ? from : AWS_EMAIL_CONFIG.SOURCE_EMAIL,
+    Source: from ? from : AWS_EMAIL_CONFIG.SOURCE_EMAIL
+  };
+
+  return new Promise((resolve, reject) => {
+    awsMailer.sendEmail(params, (error, data) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(data);
+      }
+    });
+  });
+};
+
+module.exports = {
+  sendEmail
+};

app/services/notificationService.js

@@ -1,28 +1,39 @@
-const db = require("../models/index");
-const { allMarketAlerts } = require("../helpers/db/dbHelper");
+"use strict";
+const { matchRealEstates } = require("../services/searchMatchService");
 const {
-  createMarketAlertEmailTemplate,
-  sendBulkEmail
-} = require("../helpers/awsEmail");
+  generateNotificationEmail,
+  generateNewSearchRequestEmail
+} = require("../helpers/emailContentGenerator");
+const { sendEmail } = require("../services/emailService");
 
-async function processNotifications() {
-  try {
-    const marketAlerts = await allMarketAlerts(false, false);
-    await createMarketAlertEmailTemplate();
-    if (marketAlerts.length > 0) {
-      await sendBulkEmail(marketAlerts);
-    }
+const notifyForNewRealEstates = async newRealEstates => {
+  const matches = await matchRealEstates(newRealEstates);
+  const searchRequestsToNotify = Object.keys(matches);
 
-    await db.MarketAlert.update(
-      { notified: true } /* set attributes' value */,
-      { where: { notified: false } } /* where criteria */
-    );
-  } catch (e) {
-    console.log(
-      "NOTIFICATION SERVICE: could not send notifications reason: ",
-      e
-    );
+  const asyncSendEmailActions = [];
+  for (const id of searchRequestsToNotify) {
+    const { searchRequest } = matches[id];
+    const { email } = searchRequest;
+    const allMatchingRealEstates = matches[id].realEstates || [];
+    const emailContent = generateNotificationEmail(allMatchingRealEstates, id);
+
+    const sendEmailPromise = sendEmail(email, "Nove nekretnine", emailContent);
+    asyncSendEmailActions.push(sendEmailPromise);
+    sendEmailPromise
+      .then(res => console.log(res))
+      .catch(err => console.log(err));
   }
-}
 
-module.exports = processNotifications;
+  await Promise.all(asyncSendEmailActions);
+};
+
+const notifyForNewSearchRequest = async searchRequest => {
+  const emailContent = generateNewSearchRequestEmail(searchRequest);
+  const { email } = searchRequest;
+  await sendEmail(email, "Market Alert", emailContent);
+};
+
+module.exports = {
+  notifyForNewRealEstates,
+  notifyForNewSearchRequest
+};

app/services/searchMatchService.js

@@ -0,0 +1,45 @@
+"use strict";
+
+const {
+  findSearchRequestsForRealEstate
+} = require("../helpers/db/searchRequest");
+const { addMatches } = require("../helpers/db/searchRequestMatch");
+
+const matchRealEstates = async realEstates => {
+  if (Array.isArray(realEstates)) {
+    const asyncMatchActions = [];
+    const matches = {};
+    const matchingRecords = [];
+    for (const realEstate of realEstates) {
+      const searchRequestsPromise = findSearchRequestsForRealEstate(realEstate);
+      asyncMatchActions.push(searchRequestsPromise);
+
+      searchRequestsPromise.then(searchRequests => {
+        for (const searchRequest of searchRequests) {
+          const { id } = searchRequest;
+          if (!matches[id]) {
+            matches[id] = {
+              searchRequest,
+              realEstates: []
+            };
+          }
+          matches[id].realEstates.push(realEstate);
+          matchingRecords.push({
+            searchRequestId: searchRequest.id,
+            realEstateId: realEstate.id,
+            notified: false
+          });
+        }
+      });
+    }
+
+    await Promise.all(asyncMatchActions);
+
+    await addMatches(matchingRecords);
+    return matches;
+  }
+};
+
+module.exports = {
+  matchRealEstates
+};

development.env

@@ -8,9 +8,12 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-AMAZON_ACCES_KEY_ID=(your-key-here)
-AMAZON_SECRET_ACCESS_KEY=(your-key-here)
-AMAZON_REGION=eu-west-1
+MAX_REAL_ESTATES_IN_EMAIL=Number of real estates that will be shown in URL, others will be truncated and URL with full list will be shwon
+
+AWS_KEY_ID=(your-key-here)
+AWS_SECRET_ACCESS_KEY=(your-key-here)
+AWS_REGION=eu-west-1
+
 APP_URL=http://localhost:3001
 SOURCE_EMAIL=info@saburly.com
 

index.js

@@ -1,5 +1,3 @@
-require("dotenv").config();
-
 const express = require("express");
 const path = require("path");
 const bodyParser = require("body-parser");
@@ -13,6 +11,9 @@ const {
 } = require("./app/config/appConfig");
 const routes = require("./app/routes");
 const { crawlAll } = require("./app/crawler/crawl");
+const {
+  notifyForNewRealEstates
+} = require("./app/services/notificationService");
 
 const app = express();
 
@@ -38,6 +39,7 @@ const crawl = () => {
     crawlerRunning = true;
     crawlAll().then(newRealEstates => {
       crawlerRunning = false;
+      notifyForNewRealEstates(newRealEstates);
     });
   }
 };

package-lock.json

@@ -2077,6 +2077,29 @@
         }
       }
     },
+    "he": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
+    },
+    "html-to-text": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-5.1.1.tgz",
+      "integrity": "sha512-Bci6bD/JIfZSvG4s0gW/9mMKwBRoe/1RWLxUME/d6WUSZCdY7T60bssf/jFf7EYXRyqU4P5xdClVqiYU0/ypdA==",
+      "requires": {
+        "he": "^1.2.0",
+        "htmlparser2": "^3.10.1",
+        "lodash": "^4.17.11",
+        "minimist": "^1.2.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
+        }
+      }
+    },
     "htmlparser2": {
       "version": "3.10.1",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",

package.json

@@ -11,7 +11,8 @@
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
-    "crawl": "cd app/crawler && node npmCrawl.js"
+    "crawl": "cd app/crawler && node npmCrawl.js",
+    "test-search": "cd test && node searchTest.js"
   },
   "repository": {
     "type": "git",
@@ -34,6 +35,7 @@
     "express": "^4.16.4",
     "express-ejs-layouts": "^2.5.0",
     "express-layout": "^0.1.0",
+    "html-to-text": "^5.1.1",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.26",
     "node-fetch": "^2.3.0",

test/searchTest.js

@@ -0,0 +1,24 @@
+const { getRealEstateById } = require("../app/helpers/db/realEstate");
+const {
+  notifyForNewRealEstates
+} = require("../app/services/notificationService");
+
+(async () => {
+  const realEstate1 = await getRealEstateById(53069); //B.Luka, 149.000 KM, 67m2
+  const realEstate2 = await getRealEstateById(53180); //B.Luka, 70.000 KM, 24m2
+
+  const realEstate3 = await getRealEstateById(53067); //Grbavica, 109.500 KM, 51m2
+  const realEstate4 = await getRealEstateById(53077); //alipašino, 66.000 KM, 32.58 m2
+
+  const realEstate5 = await getRealEstateById(53080); //Tuzla, - KM, 47,1 m2
+  const realEstate6 = await getRealEstateById(53646); //Tuzla, 73.500 KM, 53m2
+
+  notifyForNewRealEstates([
+    realEstate1,
+    realEstate2,
+    realEstate3,
+    realEstate4,
+    realEstate5,
+    realEstate6
+  ]);
+})();