Renamed variables to describe purpose.

app/config/appConfig.js

@@ -9,14 +9,14 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const STAGING = process.env.ENVIRONMENT !== "production";
+const USE_KIVI_ENVIRONMENT_TAG = process.env.KIVI_ENVIRONMENT !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
 const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
 const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
 
-const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10;
+const NO_CHECK_UP_DAYS = parseInt(process.env.NO_CHECK_UP_DAYS) || 10;
 
 const AWS_EMAIL_CONFIG = {
   REGION: process.env.AWS_REGION || "",
@@ -34,7 +34,7 @@ const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
 
 const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
-const API_MAP_KEY = process.env.API_MAP_KEY || "";
+const GOOGLE_MAP_KEY = process.env.GOOGLE_MAP_KEY || "";
 
 const PROSTOR_LOGIN = {
   EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
@@ -45,9 +45,6 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
-const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
-const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
-
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -58,11 +55,9 @@ module.exports = {
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
   PRINT_CRAWLER_DEBUG,
-  API_MAP_KEY,
-  STAGING,
-  CHECK_UP_DAYS,
+  GOOGLE_MAP_KEY,
+  USE_KIVI_ENVIRONMENT_TAG,
+  NO_CHECK_UP_DAYS,
   PROSTOR_LOGIN,
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY
+  USER_AGENT
 };

app/controllers/location.js

@@ -17,15 +17,15 @@ const getLocation = async (req, res) => {
     return;
   }
   const selectedArea = searchRequest.areaToSearch;
-  const sw = selectedArea.coordinates[0][3];
-  const ne = selectedArea.coordinates[0][1];
+  const southWest = selectedArea.coordinates[0][3];
+  const northEast = selectedArea.coordinates[0][1];
 
   if (sw[0] && ne[0]) {
     selectedLatLngBounds = {
-      swLat: sw[1],
-      swLng: sw[0],
-      neLat: ne[1],
-      neLng: ne[0]
+      swLat: southWest[1],
+      swLng: southWest[0],
+      neLat: northEast[1],
+      neLng: northEast[0]
     };
     boundsSelected = true;
   }

app/helpers/db/searchRequest.js

@@ -3,7 +3,6 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -17,22 +16,6 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
-const findAllRequestsForCheckUp = async () => {
-  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
-  const checkupDate = new Date();
-  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
-
-  const dateQuery = {
-    notifiedAt: {
-      [Op.lte]: checkupDate
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: dateQuery
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -476,6 +459,5 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate,
-  findAllRequestsForCheckUp
+  findSearchRequestsForRealEstate
 };

app/helpers/db/searchRequestMatch.js

@@ -2,6 +2,7 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
+const { NO_CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -42,6 +43,42 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
+const findAllRequestsForCheckUp = async () => {
+  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
+  //The ones that received notification for real estate NO_CHECK_UP_DAYS days from now
+  const date = new Date();
+  const checkUpDate = date.getDate() - NO_CHECK_UP_DAYS;
+  date.setDate(checkUpDate);
+  const dateQuery = {
+    createdAt: {
+      [Op.gte]: date
+    }
+  };
+
+  const excludedMatches = await db.SearchRequestMatch.findAll({
+    attributes: ["searchRequestId"],
+    where: dateQuery,
+    order: [["searchRequestId", "ASC"]]
+  });
+
+  const excludedRequestsAll = excludedMatches.map(match => {
+    return match.dataValues.searchRequestId;
+  });
+  //Removing duplicate search request id-s for optimization
+  const excludedRequests = [...new Set(excludedRequestsAll)];
+
+  const query = {
+    subscribed: true,
+    id: {
+      [Op.notIn]: excludedRequests
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: query
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -52,5 +89,6 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp
 };

app/helpers/emailContentGenerator.js

@@ -3,12 +3,12 @@
 const {
   MAX_REAL_ESTATES_IN_EMAIL,
   APP_URL,
-  STAGING
+  USE_KIVI_ENVIRONMENT_TAG
 } = require("../config/appConfig");
 const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-//Tag to recognize staging from development
-const stagingTag = STAGING ? "[STAGING] " : "";
+//Tag to recognize staging from development if needed
+const stagingTag = USE_KIVI_ENVIRONMENT_TAG ? "[STAGING] " : "";
 
 const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
   return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>

app/helpers/fetchWrapper.js

@@ -1,9 +1,5 @@
 const nodeFetch = require("node-fetch");
-const {
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY
-} = require("../config/appConfig");
+const { USER_AGENT } = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
   const newOptions = Object.assign({}, options);
@@ -11,11 +7,7 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
-  const urlAdaptedForScraping = USE_SCRAPER_API
-    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
-    : url;
-
-  return nodeFetch(urlAdaptedForScraping, newOptions);
+  return nodeFetch(url, newOptions);
 };
 
 module.exports = fetch;

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -1,14 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
-      type: Sequelize.DATE,
-      defaultValue: new Date()
-    });
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
-  }
-};

app/models/searchRequest.js

@@ -82,11 +82,7 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT,
-    notifiedAt: {
-      type: DataTypes.DATE,
-      defaultValue: new Date()
-    }
+    heatingType: DataTypes.TEXT
   });
 
   return SearchRequest;

app/services/notificationService.js

@@ -1,7 +1,7 @@
 "use strict";
-const { STAGING } = require("../config/appConfig");
+const { USE_KIVI_ENVIRONMENT_TAG } = require("../config/appConfig");
 
-const stagingTag = STAGING ? "[STAGING] " : "";
+const stagingTag = USE_KIVI_ENVIRONMENT_TAG ? "[STAGING] " : "";
 
 const {
   matchRealEstates,
@@ -15,10 +15,9 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
+  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
-const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
-
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -36,7 +35,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
+
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -77,10 +76,6 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
-
-        //Change time of notified At for searchReq
-        searchRequest.notifiedAt = new Date();
-        searchRequest.save();
       }
     }
   }
@@ -136,7 +131,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -148,12 +143,8 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
-
-    //Change time of notified At for searchReq
-    searchRequest.notifiedAt = new Date();
-    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions);
+  await Promise.all(asyncSendEmailActions); */
 };
 
 module.exports = {

app/views/location.ejs

@@ -217,7 +217,7 @@
   });
 </script>
 <script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
+  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
   async
   defer
 ></script>

development.env

@@ -8,23 +8,19 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-ENVIRONMENT=Variable to denote development, staging and production
+KIVI_ENVIRONMENT=Variable to denote development, staging and production
 
 USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
 
-CHECK_UP_DAYS=Check up email is sent after this number of days without notification
+NO_CHECK_UP_DAYS=Check up email is sent after this number of days without notification
 #=============== GOOGLE ANALYTICS =============#
 GA_ID=Google Analytics ID
 
 #=============== GOOGLE MAPS =============#
-API_MAP_KEY=(your-key-here)
-
-#=============== SCRAPER API SUPORT =============#
-USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api 
+GOOGLE_MAP_KEY=(your-key-here)
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)