Merge branch 'add-email-frequency-option' into 'master'

Add email frequency option

See merge request saburly/marketalarm/web!68

app/common/enums.js

@@ -174,10 +174,24 @@ const CRAWLER_AD_TYPE = {
   ONLY_REQUEST: 4
 };
 
+const EMAIL_FREQUENCY = {
+  ASAP: {
+    id: 1,
+    stringId: "ASAP",
+    title: "Odmah"
+  },
+  DAILY: {
+    id: 2,
+    stringId: "DAILY",
+    title: "Jednom dnevno"
+  }
+};
+
 module.exports = {
   AD_TYPE,
   AD_CATEGORY,
   AD_STATUS,
   AD_AGENCY,
-  CRAWLER_AD_TYPE
+  CRAWLER_AD_TYPE,
+  EMAIL_FREQUENCY
 };

app/controllers/queryReview.js

@@ -3,9 +3,9 @@ const { isValidEmail } = require("../helpers/email");
 const {
   notifyForNewSearchRequest
 } = require("../services/notificationService");
-const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
+const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-const getQueryReviewData = searchRequest => {
+const getQueryReviewTableData = searchRequest => {
   const {
     id,
     adType,
@@ -87,15 +87,26 @@ const getQueryReview = async (req, res) => {
   const title = "Da li je ovo to što ste tražili ?";
   const nextStep = req.query.nextStep;
   const error = req.query.error;
-  const queryReviewData = getQueryReviewData(searchRequest);
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
   const email = searchRequest.email;
+  let selectedEmailFrequency;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.ASAP.id;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.DAILY.id;
+      break;
+  }
 
   res.render("queryReview", {
     nextStep,
-    queryReviewData,
+    queryReviewTableData,
     title,
     email,
-    error
+    selectedEmailFrequency,
+    error,
+    EMAIL_FREQUENCY
   });
 };
 
@@ -107,17 +118,26 @@ const postQueryReview = async (req, res) => {
   }
 
   const nextStep = req.query.nextStep || "/ponovo";
+  const emailFrequency =
+    parseInt(req.body.emailFrequency) || EMAIL_FREQUENCY.ASAP.id;
   const emailInput = req.body.email;
   const emailConfirmInput = req.body.confirmEmail;
   const title = "Da li je ovo to što ste tražili ?";
-  const queryReviewData = getQueryReviewData(searchRequest);
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
+
+  let emailFrequencyStringId = EMAIL_FREQUENCY.ASAP.stringId;
+  if (emailFrequency === EMAIL_FREQUENCY.DAILY.id) {
+    emailFrequencyStringId = EMAIL_FREQUENCY.DAILY.stringId;
+  }
+
+  searchRequest.emailFrequency = emailFrequencyStringId;
 
   if (emailInput !== emailConfirmInput) {
     const error = "Greška ! Unešeni emailovi nisu isti";
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;
@@ -128,7 +148,7 @@ const postQueryReview = async (req, res) => {
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;
@@ -147,7 +167,7 @@ const postQueryReview = async (req, res) => {
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;
@@ -164,7 +184,7 @@ const postQueryReview = async (req, res) => {
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;

app/helpers/db/searchRequestMatch.js

@@ -24,6 +24,23 @@ const findRealEstatesForSearchRequest = async searchRequestId => {
   return matchingRealEstates;
 };
 
+const findNotNotifiedMatches = async () => {
+  const query = {
+    notified: false
+  };
+
+  const searchRequestsModel = { model: db.SearchRequest, as: "searchRequests" };
+  const realEstateModel = { model: db.RealEstate, as: "realEstates" };
+  const include = [searchRequestsModel, realEstateModel];
+
+  const matchingRecords = await db.SearchRequestMatch.findAll({
+    where: query,
+    include
+  });
+
+  return matchingRecords;
+};
+
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
     ignoreDuplicates: true
@@ -32,5 +49,6 @@ const addMatches = async matchingRecords => {
 
 module.exports = {
   findRealEstatesForSearchRequest,
-  addMatches
+  addMatches,
+  findNotNotifiedMatches
 };

app/helpers/emailContentGenerator.js

@@ -20,7 +20,11 @@ const generateRealEstateLinks = realEstates => {
   return realEstateLinks;
 };
 
-const generateNotificationEmail = (realEstates, searchRequestId) => {
+const generateNotificationEmail = (
+  realEstates,
+  searchRequestId,
+  dailyNotification = false
+) => {
   const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
   const realEstatesToShow = truncateList
     ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
@@ -30,9 +34,20 @@ const generateNotificationEmail = (realEstates, searchRequestId) => {
   const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
   const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
   const emailFooter = generateEmailFooter(searchRequestId);
+  const asapMessageBody =
+    realEstates.length > 1
+      ? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
+      : "Pronašli smo nekretninu koja odgovara Vašoj pretrazi";
+
+  const dailyMessageBody =
+    realEstates.length > 1
+      ? "U posljednja 24h objavljene su sljedeće nekretnine koje odgovaraju uslovima Vaše pretrage"
+      : "U posljednja 24h objavljena je sljedeća nekretnina koja odgovara uslovima Vaše pretrage";
+
+  const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
 
   return `<h3>Zdravo</h3>
-  <h4>Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi</h4>
+  <h4>${messageBody}</h4>
   <div>
   ${realEstateLinks}
   <div/>

app/migrations/20191101113541-add-emailFrequency-column-to-searchRequest-table.js

@@ -0,0 +1,15 @@
+"use strict";
+const { EMAIL_FREQUENCY } = require("../common/enums");
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "emailFrequency", {
+      type: Sequelize.TEXT,
+      defaultValue: EMAIL_FREQUENCY.ASAP.stringId
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "emailFrequency");
+  }
+};

app/models/searchRequest.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const { AD_TYPE } = require("../common/enums");
+const { AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
 module.exports = (sequelize, DataTypes) => {
   const SearchRequest = sequelize.define("SearchRequest", {
@@ -61,6 +61,11 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.BOOLEAN,
       defaultValue: false,
       allowNull: false
+    },
+    emailFrequency: {
+      type: DataTypes.TEXT,
+      defaultValue: EMAIL_FREQUENCY.ASAP.stringId,
+      allowNull: false
     }
   });
 

app/models/searchRequestMatch.js

@@ -44,6 +44,12 @@ module.exports = (sequelize, DataTypes) => {
   );
 
   SearchRequestMatch.associate = models => {
+    SearchRequestMatch.hasMany(models.SearchRequest, {
+      foreignKey: "id",
+      sourceKey: "searchRequestId",
+      targetKey: "id",
+      as: "searchRequests"
+    });
     SearchRequestMatch.hasMany(models.RealEstate, {
       foreignKey: "id",
       as: "realEstates"

app/npmScripts/npmDailyNotify.js

@@ -0,0 +1,8 @@
+"use strict";
+const {
+  notifyRequestsWithDailyOption
+} = require("../services/notificationService");
+
+(async () => {
+  await notifyRequestsWithDailyOption();
+})();

app/public/segment.css

@@ -1,14 +1,18 @@
-.ui-segment {
+.segmented {
   color: #02adba;
   border: 1px solid #02adba;
   border-radius: 4px;
   display: inline-block;
 }
-.ui-segment span.option.active {
+.segmented label {
+  color: #02adba;
+}
+.segmented input:checked + .label {
   background-color: #02adba;
   color: white;
 }
-.ui-segment span.option {
+[type="radio"]:not(:checked) + span,
+[type="radio"]:checked + span {
   padding-left: 30px;
   padding-right: 30px;
   height: 35px;
@@ -21,9 +25,14 @@
   border-right: 1px solid #02adba;
 }
 
-.ui-segment span.option:last-child {
+.segmented :last-child .label {
   border-right: none;
 }
-.segment-select {
+.segmented input {
+  display: none;
+}
+
+span.label:before,
+span.label:after {
   display: none;
 }

app/services/notificationService.js

@@ -8,6 +8,7 @@ const {
   generateNewSearchRequestEmail,
   generateEmailSubject
 } = require("../helpers/emailContentGenerator");
+const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -29,34 +30,87 @@ const notifyForNewSearchRequest = async searchRequest => {
   await sendEmail(email, "Kivi - novi zahtjev za pretragu", emailContent);
 };
 
-const notifyMatches = async matches => {
+const notifyMatches = async (matches, dailyNotification = false) => {
   const searchRequestsToNotify = Object.keys(matches);
 
   const asyncSendEmailActions = [];
   for (const id of searchRequestsToNotify) {
-    const { searchRequest } = matches[id];
+    const { searchRequest, notifyNow } = matches[id];
-    const { email } = searchRequest;
+    if (notifyNow) {
-    const allMatchingRealEstates = matches[id].realEstates || [];
+      const { email } = searchRequest;
-    if (allMatchingRealEstates.length > 0) {
+      const allMatchingRealEstates = matches[id].realEstates || [];
-      const emailContent = generateNotificationEmail(
+      if (allMatchingRealEstates.length > 0) {
-        allMatchingRealEstates,
+        const emailContent = generateNotificationEmail(
-        id
+          allMatchingRealEstates,
-      );
+          id,
-      const emailSubject = generateEmailSubject(
+          dailyNotification
-        allMatchingRealEstates.length,
+        );
-        allMatchingRealEstates[0].title
+        const emailSubject = generateEmailSubject(
-      );
+          allMatchingRealEstates.length,
+          allMatchingRealEstates[0].title
+        );
 
-      const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
+        const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
-      asyncSendEmailActions.push(sendEmailPromise);
+        asyncSendEmailActions.push(sendEmailPromise);
-      sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+        sendEmailPromise.catch(err =>
+          console.log("[Email Sending Failed]", err)
+        );
+      }
     }
   }
 
   await Promise.all(asyncSendEmailActions);
 };
 
+const notifyRequestsWithDailyOption = async () => {
+  const notNotifiedSearchRequestMatches = await findNotNotifiedMatches();
+
+  const matches = {};
+
+  for (const searchRequestMatch of notNotifiedSearchRequestMatches) {
+    const { searchRequests, realEstates } = searchRequestMatch;
+
+    if (!Array.isArray(searchRequests) || searchRequests.length !== 1) {
+      // Something is wrong with this match
+      // (search request not found for specified search request id)
+      // OR
+      // there are multiple search requests with the same ID (this should never be the case !
+      // TODO: Maybe if association is defined better, this will be automatically only one object instead of array
+      continue;
+    }
+
+    if (!Array.isArray(realEstates) || realEstates.length !== 1) {
+      // Something is wrong with this match
+      // (real estate not found for specified real estate id)
+      // OR
+      // there are multiple real estates with the same ID (this should never be the case !
+      // TODO: Maybe if association is defined better, this will be automatically only one object instead of array
+      continue;
+    }
+
+    const searchRequest = searchRequests[0];
+    const realEstate = realEstates[0];
+    const searchRequestId = searchRequest.id;
+
+    if (!matches[searchRequestId]) {
+      matches[searchRequestId] = {
+        searchRequest,
+        realEstates: [],
+        notifyNow: true
+      };
+    }
+
+    matches[searchRequestId].realEstates.push(realEstate);
+
+    searchRequestMatch.notified = true;
+    searchRequestMatch.save();
+  }
+
+  await notifyMatches(matches, true);
+};
+
 module.exports = {
   notifyForNewRealEstates,
-  notifyForNewSearchRequest
+  notifyForNewSearchRequest,
+  notifyRequestsWithDailyOption
 };

app/services/searchMatchService.js

@@ -6,6 +6,7 @@ const {
 const { findRealEstatesForSearchRequest } = require("../helpers/db/realEstate");
 const { addMatches } = require("../helpers/db/searchRequestMatch");
 const { MAX_REAL_ESTATES_IN_FIRST_EMAIL } = require("../config/appConfig");
+const { EMAIL_FREQUENCY } = require("../common/enums");
 
 const matchRealEstates = async realEstates => {
   if (Array.isArray(realEstates)) {
@@ -18,18 +19,19 @@ const matchRealEstates = async realEstates => {
 
       searchRequestsPromise.then(searchRequests => {
         for (const searchRequest of searchRequests) {
-          const { id } = searchRequest;
+          const { id, emailFrequency } = searchRequest;
           if (!matches[id]) {
             matches[id] = {
               searchRequest,
-              realEstates: []
+              realEstates: [],
+              notifyNow: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
             };
           }
           matches[id].realEstates.push(realEstate);
           matchingRecords.push({
             searchRequestId: searchRequest.id,
             realEstateId: realEstate.id,
-            notified: false
+            notified: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
           });
         }
       });
@@ -62,7 +64,7 @@ const matchSearchRequest = async searchRequest => {
     matchingRecords.push({
       searchRequestId,
       realEstateId: realEstate.id,
-      notified: false
+      notified: true
     });
   }
 

app/views/queryReview.ejs

@@ -2,7 +2,7 @@
 <form method="POST" id="form-queryreview">
     <div class="row center-align">
       <ul class="collection with-header">
-        <% for(const stepData of queryReviewData) { %>
+        <% for(const stepData of queryReviewTableData) { %>
           <li class="collection-item" >
             <div id="<%= stepData.id %>" ><%= stepData.title || '-' %>
               <a href="<%= stepData.url %>" class="kivi-color secondary-content">
@@ -13,6 +13,26 @@
         <% } %>
       </ul>
     </div>
+    <div class="row center-align">
+        <h6>Slanje obavještenja</h6>
+        <span class="segmented">
+                <label>
+                    <input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.ASAP.id %>"
+                    <% if (selectedEmailFrequency === EMAIL_FREQUENCY.ASAP.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= EMAIL_FREQUENCY.ASAP.title %></span>
+                </label>
+
+                <label>
+                    <input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.DAILY.id %>"
+                    <% if (selectedEmailFrequency === EMAIL_FREQUENCY.DAILY.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= EMAIL_FREQUENCY.DAILY.title %></span>
+                </label>
+		    </span>
+    </div>
     <div class="row center-align">
         <div class="col">
             <input id="email" name="email" type="email" placeholder="vas.email@mail.com" <% if (email) { %>value="<%= email %>" <% } %> required size="250" />
@@ -47,6 +67,7 @@
 </form>
 
 <script>
+
  $(document).ready( () => {
    $("#submit").click( () => {
        const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;

app/views/realEstateType.ejs

@@ -3,18 +3,23 @@
     <div class="center-align">
 
         <div class="row">
-            <select class="segment-select" id="adType" name="adType">
+            <span class="segmented">
-                <option value="<%= AD_TYPE.AD_TYPE_SALE.id %>"
+                <label>
-                <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
+                    <input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_SALE.id %>"
-                    selected="selected"
+                    <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
-                <% } %>
+                    checked
-                ><%= AD_TYPE.AD_TYPE_SALE.title %></option>
+                    <% } %>>
-                <option value="<%= AD_TYPE.AD_TYPE_RENT.id %>"
+                    <span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
-                <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
+                </label>
-                    selected="selected"
+
-                <% } %>
+                <label>
-                ><%= AD_TYPE.AD_TYPE_RENT.title %></option>
+                    <input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_RENT.id %>"
-            </select>
+                    <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
+                </label>
+		    </span>
         </div>
 
         <br>
@@ -37,45 +42,6 @@
 </form>
 
 <script>
-    (function($) {
-        $.fn.extend({
-            Segment: function() {
-                $(this).each(function() {
-                    const self = $(this);
-                    const onchange = self.attr('onchange');
-                    const wrapper = $("<div>", { class: "ui-segment" });
-                    $(this)
-                        .find("option")
-                        .each(function() {
-                            const option = $("<span>", {
-                                class: "option",
-                                onclick: onchange,
-                                text:  $(this).text(),
-                                value: $(this).val(),
-                            });
-                            if ($(this).is(":selected")) {
-                                option.addClass("active");
-                            }
-                            wrapper.append(option);
-                        });
-
-                    wrapper.find("span.option").click(function (){
-                        wrapper.find("span.option").removeClass("active");
-                        $(this).addClass("active");
-                        self.val($(this).attr('value'));
-                    });
-
-                    $(this).after(wrapper);
-                    $(this).hide();
-                });
-            }
-        });
-    })(jQuery);
-
-    $(document).ready(() => {
-        $(".segment-select").Segment();
-    });
-
   function saveAndSubmit(id) {
    $("#realEstateType").val(id);
    $("#realEstateTypeSelection > a").attr("onclick", "");

development.env

@@ -31,6 +31,7 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
 OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
 OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==RENTAL==
 RENTAL_MAX_PAGES=Restrict crawler to this number of pages
 RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -38,6 +39,7 @@ RENTAL_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check co
 RENTAL_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 RENTAL_IGNORED_USERNAMES=!!! This is not used for rental crawler !!!
 RENTAL_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+RENTAL_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==PROSTOR==
 PROSTOR_MAX_PAGES=!!! This is not used for prostor crawler !!!
 PROSTOR_MAX_RESULTS_PER_PAGE=For Prostor crawler, this represents MAX RESULTS in total
@@ -45,6 +47,7 @@ PROSTOR_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check c
 PROSTOR_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
 PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
+PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==AKTIDO==
 AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
 AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -52,3 +55,4 @@ AKTIDO_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check co
 AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

package.json

@@ -12,6 +12,7 @@
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
     "crawl": "cd app/crawler && node npmCrawl.js",
+    "daily-notify": "cd app/npmScripts && node npmDailyNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js"
   },