Merge branch 'master' into 'checkup-email-bug-fix'

# Conflicts:
#   app/services/notificationService.js

app/config/appConfig.js

@@ -45,6 +45,9 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -59,5 +62,7 @@ module.exports = {
   STAGING,
   CHECK_UP_DAYS,
   PROSTOR_LOGIN,
-  USER_AGENT
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
 };

app/helpers/db/searchRequest.js

@@ -3,6 +3,7 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -16,6 +17,22 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
+const findAllRequestsForCheckUp = async () => {
+  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
+  const checkupDate = new Date();
+  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
+
+  const dateQuery = {
+    notifiedAt: {
+      [Op.lte]: checkupDate
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: dateQuery
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -156,11 +173,33 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       // If real estate dont have defined number of rooms ex. null
-      //It returns all search requests except for ones that dont want incpomlete ads
+      //It returns requests that didn't choose number of rooms - also null
+      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            [Op.and]: [
+              {
+                numberOfRoomsMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                numberOfRoomsMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
       };
     }
   }
@@ -190,9 +229,30 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       numberOfFloorsQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            [Op.and]: [
+              {
+                numberOfFloorsMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                numberOfFloorsMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
       };
     }
   }
@@ -221,9 +281,30 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       floorQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            [Op.and]: [
+              {
+                floorMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                floorMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
       };
     }
   }
@@ -232,7 +313,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
   //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
   //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
   //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
-  //If real estate characteristic = null, dont know if true or false, find all req except ones that dont want incomplete ads
+  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
   let balconyQuery = {};
   if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
     if (balcony === false) {
@@ -243,9 +324,30 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (balcony === null) {
       balconyQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            balcony: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                balcony: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
+                }
+              }
+            ]
+          }
+        ]
       };
     }
   }
@@ -259,9 +361,30 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (newBuilding === null) {
       newBuildingQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            newBuilding: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                newBuilding: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
+                }
+              }
+            ]
+          }
+        ]
       };
     }
   }
@@ -275,13 +398,33 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (elevator === null) {
       elevatorQuery = {
-        includeIncompleteAds: {
+        [Op.or]: [
-          [Op.ne]: false
+          {
-        }
+            elevator: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                elevator: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
+                }
+              }
+            ]
+          }
+        ]
       };
     }
   }
-
   //General query consists of each individual query
   const query = {
     adType,
@@ -333,5 +476,6 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate
+  findSearchRequestsForRealEstate,
+  findAllRequestsForCheckUp
 };

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -43,42 +42,6 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
-const findAllRequestsForCheckUp = async () => {
-  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate CHECK_UP_DAYS days from now
-  const date = new Date();
-  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
-  date.setDate(checkUpDate);
-  const dateQuery = {
-    createdAt: {
-      [Op.gte]: date
-    }
-  };
-
-  const excludedMatches = await db.SearchRequestMatch.findAll({
-    attributes: ["searchRequestId"],
-    where: dateQuery,
-    order: [["searchRequestId", "ASC"]]
-  });
-
-  const excludedRequestsAll = excludedMatches.map(match => {
-    return match.dataValues.searchRequestId;
-  });
-  //Removing duplicate search request id-s for optimization
-  const excludedRequests = [...new Set(excludedRequestsAll)];
-
-  const query = {
-    subscribed: true,
-    id: {
-      [Op.notIn]: excludedRequests
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: query
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -89,6 +52,5 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches,
+  findNotNotifiedMatches
-  findAllRequestsForCheckUp
 };

app/helpers/fetchWrapper.js

@@ -1,5 +1,9 @@
 const nodeFetch = require("node-fetch");
-const { USER_AGENT } = require("../config/appConfig");
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
+} = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
   const newOptions = Object.assign({}, options);
@@ -7,7 +11,11 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
-  return nodeFetch(url, newOptions);
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    : url;
+
+  return nodeFetch(urlAdaptedForScraping, newOptions);
 };
 
 module.exports = fetch;

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
+      type: Sequelize.DATE,
+      defaultValue: new Date()
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
+  }
+};

app/models/searchRequest.js

@@ -82,7 +82,11 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT
+    heatingType: DataTypes.TEXT,
+    notifiedAt: {
+      type: DataTypes.DATE,
+      defaultValue: new Date()
+    }
   });
 
   return SearchRequest;

app/services/notificationService.js

@@ -15,9 +15,10 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
-  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
+
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -35,7 +36,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-
+  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -76,6 +77,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
+
+        //Change time of notified At for searchReq
+        searchRequest.notifiedAt = new Date();
+        searchRequest.save();
       }
     }
   }
@@ -131,7 +136,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+  const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -143,8 +148,12 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+
+    //Change time of notified At for searchReq
+    searchRequest.notifiedAt = new Date();
+    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions); */
+  await Promise.all(asyncSendEmailActions);
 };
 
 module.exports = {

development.env

@@ -22,6 +22,10 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api 
+
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)