Merge branch 'master' into 'checkup-email-bug-fix'

# Conflicts:
#   app/services/notificationService.js

app/config/appConfig.js

@@ -9,7 +9,7 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const STAGING = process.env.SETTINGS !== "production";
+const STAGING = process.env.ENVIRONMENT !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
@@ -41,6 +41,13 @@ const PROSTOR_LOGIN = {
   PASSWORD: process.env.PROSTOR_LOGIN_PASS
 };
 
+const USER_AGENT =
+  process.env.USER_AGENT ||
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
+
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -54,6 +61,8 @@ module.exports = {
   API_MAP_KEY,
   STAGING,
   CHECK_UP_DAYS,
-  PROSTOR_LOGIN
-
+  PROSTOR_LOGIN,
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
 };

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/olx.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/prostor.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
@@ -191,13 +191,7 @@ class ProstorCrawler {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
 
     //Status information is given already in realestate list
-    //For VIP Ads status ='' canot be used, but no VIP ads are crawled
-    //We will make "fake" vip ad for RE that have size=55
-    //It is weird because yesterday it said 'VIP ponuda' ???
-    const adStatus =
-      size === "55"
-        ? ProstorCrawler.getStatusId("VIP ponuda")
-        : ProstorCrawler.getStatusId(status);
+    const adStatus = ProstorCrawler.getStatusId(status);
 
     const url = `https://prostor.ba${link}`;
 

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -399,7 +399,9 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
           };
         }
 
@@ -410,7 +412,9 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 
@@ -268,6 +268,7 @@ class SaljicCrawler {
 
       const descriptions = $(propertySelectors.descriptions)
         .text()
+        .replace(/\"/g, "")
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
@@ -283,8 +284,8 @@ class SaljicCrawler {
       const locationLong = parseFloat(longText) || null;
 
       //====== DETAIL INFORMATION FIELDS ==========
-      let area,
-        gardenSize,
+      let area = null,
+        gardenSize = null,
         numberOfRooms = null,
         numberOfFloors = null,
         floor = null,
@@ -315,6 +316,7 @@ class SaljicCrawler {
         includingBills = null,
         animalsAllowed = null,
         pool = null,
+        exchange = null,
         urbanPlanPermit = null,
         buildingPermit = null,
         utilityConnection = null,
@@ -323,7 +325,14 @@ class SaljicCrawler {
       let renewedDate = null;
       let realEstateType;
       let numberOfViewsAgency = null;
-
+      let numberOfViewsKivi = null;
+      let streetNumber = 0;
+      let adStatus = status;
+      let shortDescription = descriptions.substring(
+        0,
+        descriptions.indexOf(".")
+      );
+      let longDescription = descriptions;
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -482,29 +491,37 @@ class SaljicCrawler {
         renewedDate = new Date();
       }
 
+      const originAgencyName = AD_AGENCY.SALJIC;
+      const locality = "";
+      const municipality = "";
+      const city = "";
+      const region = "";
+      const entity = "";
+      const country = "";
+
       const data = {
         url,
         agencyObjectId,
-        originAgencyName: AD_AGENCY.SALJIC,
+        originAgencyName,
         realEstateType,
         adType,
         title,
         price,
         area,
         gardenSize,
-        shortDescription: descriptions.substring(0, descriptions.indexOf(".")),
-        longDescription: descriptions,
-        streetNumber: 0,
+        shortDescription,
+        longDescription,
+        streetNumber,
         streetName,
-        locality: "",
-        municipality: "",
-        city: "",
-        region: "",
-        entity: "",
-        country: "",
+        locality,
+        municipality,
+        city,
+        region,
+        entity,
+        country,
         locationLat,
         locationLong,
-        adStatus: status,
+        adStatus,
         publishedDate,
         renewedDate,
         numberOfRooms,
@@ -537,12 +554,15 @@ class SaljicCrawler {
         includingBills,
         animalsAllowed,
         pool,
+        exchange,
         urbanPlanPermit,
         buildingPermit,
         utilityConnection,
         distanceToRiver,
-        numberOfViewsAgency
+        numberOfViewsAgency,
+        numberOfViewsKivi
       };
+
       return data;
     } catch (e) {
       console.error("Exception caught: " + e.message, "\r\nURL:", url);

app/helpers/db/realEstate.js

@@ -332,10 +332,14 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
+  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where: includeIncompleteAds ? queryIncludeIncomplete : query,
+    where:
+      includeIncompleteAds || includeIncompleteAds == null
+        ? queryIncludeIncomplete
+        : query,
     limit: maxResults,
     order
   });

app/helpers/db/searchRequest.js

@@ -3,6 +3,7 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -16,6 +17,22 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
+const findAllRequestsForCheckUp = async () => {
+  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
+  const checkupDate = new Date();
+  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
+
+  const dateQuery = {
+    notifiedAt: {
+      [Op.lte]: checkupDate
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: dateQuery
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -157,7 +174,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
+      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -176,7 +193,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -226,7 +246,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -275,7 +298,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -313,7 +339,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -347,7 +376,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -381,7 +413,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -423,10 +458,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query
+  //Tag to check if incomplete ads are accepted in query which is default
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.eq]: true
+      [Op.or]: {
+        [Op.eq]: true,
+        [Op.is]: null
+      }
     };
   }
 
@@ -438,5 +476,6 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate
+  findSearchRequestsForRealEstate,
+  findAllRequestsForCheckUp
 };

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -43,42 +42,6 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
-const findAllRequestsForCheckUp = async () => {
-  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate CHECK_UP_DAYS days from now
-  const date = new Date();
-  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
-  date.setDate(checkUpDate);
-  const dateQuery = {
-    createdAt: {
-      [Op.gte]: date
-    }
-  };
-
-  const excludedMatches = await db.SearchRequestMatch.findAll({
-    attributes: ["searchRequestId"],
-    where: dateQuery,
-    order: [["searchRequestId", "ASC"]]
-  });
-
-  const excludedRequestsAll = excludedMatches.map(match => {
-    return match.dataValues.searchRequestId;
-  });
-  //Removing duplicate search request id-s for optimization
-  const excludedRequests = [...new Set(excludedRequestsAll)];
-
-  const query = {
-    subscribed: true,
-    id: {
-      [Op.notIn]: excludedRequests
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: query
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -89,6 +52,5 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches,
-  findAllRequestsForCheckUp
+  findNotNotifiedMatches
 };

app/helpers/fetchWrapper.js

@@ -0,0 +1,21 @@
+const nodeFetch = require("node-fetch");
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
+} = require("../config/appConfig");
+
+const fetch = async (url, options = {}) => {
+  const newOptions = Object.assign({}, options);
+  if (!newOptions["headers"]) {
+    newOptions["headers"] = {};
+  }
+  newOptions["headers"]["User-Agent"] = USER_AGENT;
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    : url;
+
+  return nodeFetch(urlAdaptedForScraping, newOptions);
+};
+
+module.exports = fetch;

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
+      type: Sequelize.DATE,
+      defaultValue: new Date()
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
+  }
+};

app/models/searchRequest.js

@@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [
-          [
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0]
-          ]
-        ],
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -90,7 +82,11 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT
+    heatingType: DataTypes.TEXT,
+    notifiedAt: {
+      type: DataTypes.DATE,
+      defaultValue: new Date()
+    }
   });
 
   return SearchRequest;

app/services/notificationService.js

@@ -15,9 +15,10 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
-  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
+
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -35,7 +36,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-
+  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -76,6 +77,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
+
+        //Change time of notified At for searchReq
+        searchRequest.notifiedAt = new Date();
+        searchRequest.save();
       }
     }
   }
@@ -143,6 +148,10 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+
+    //Change time of notified At for searchReq
+    searchRequest.notifiedAt = new Date();
+    searchRequest.save();
   }
   await Promise.all(asyncSendEmailActions);
 };

app/views/advancedFilters.ejs

@@ -61,9 +61,8 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
-      <% if (includeIncompleteAds) { %>
            checked
-      <% } %>>
+      >
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

development.env

@@ -8,8 +8,9 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-SETTINGS=Variable to denote development, staging and production
+ENVIRONMENT=Variable to denote development, staging and production
 
+USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
@@ -21,6 +22,10 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api 
+
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)
@@ -69,4 +74,4 @@ AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found