More elegant scrape of lat and long.

app/config/appConfig.js

@@ -9,14 +9,14 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const USE_KIVI_ENVIRONMENT_TAG = process.env.KIVI_ENVIRONMENT !== "production";
+const STAGING = process.env.ENVIRONMENT !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
 const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
 const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
 
-const NO_CHECK_UP_DAYS = parseInt(process.env.NO_CHECK_UP_DAYS) || 10;
+const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10;
 
 const AWS_EMAIL_CONFIG = {
   REGION: process.env.AWS_REGION || "",
@@ -34,17 +34,13 @@ const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
 
 const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
-const GOOGLE_MAP_KEY = process.env.GOOGLE_MAP_KEY || "";
+const API_MAP_KEY = process.env.API_MAP_KEY || "";
 
 const PROSTOR_LOGIN = {
   EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
   PASSWORD: process.env.PROSTOR_LOGIN_PASS
 };
 
-const USER_AGENT =
-  process.env.USER_AGENT ||
-  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
-
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -55,9 +51,8 @@ module.exports = {
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
   PRINT_CRAWLER_DEBUG,
-  GOOGLE_MAP_KEY,
-  USE_KIVI_ENVIRONMENT_TAG,
-  NO_CHECK_UP_DAYS,
-  PROSTOR_LOGIN,
-  USER_AGENT
+  API_MAP_KEY,
+  STAGING,
+  CHECK_UP_DAYS,
+  PROSTOR_LOGIN
 };

app/controllers/location.js

@@ -17,15 +17,15 @@ const getLocation = async (req, res) => {
     return;
   }
   const selectedArea = searchRequest.areaToSearch;
-  const southWest = selectedArea.coordinates[0][3];
-  const northEast = selectedArea.coordinates[0][1];
+  const sw = selectedArea.coordinates[0][3];
+  const ne = selectedArea.coordinates[0][1];
 
   if (sw[0] && ne[0]) {
     selectedLatLngBounds = {
-      swLat: southWest[1],
-      swLng: southWest[0],
-      neLat: northEast[1],
-      neLng: northEast[0]
+      swLat: sw[1],
+      swLng: sw[0],
+      neLat: ne[1],
+      neLng: ne[0]
     };
     boundsSelected = true;
   }

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/olx.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/prostor.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
@@ -191,7 +191,13 @@ class ProstorCrawler {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
 
     //Status information is given already in realestate list
-    const adStatus = ProstorCrawler.getStatusId(status);
+    //For VIP Ads status ='' canot be used, but no VIP ads are crawled
+    //We will make "fake" vip ad for RE that have size=55
+    //It is weird because yesterday it said 'VIP ponuda' ???
+    const adStatus =
+      size === "55"
+        ? ProstorCrawler.getStatusId("VIP ponuda")
+        : ProstorCrawler.getStatusId(status);
 
     const url = `https://prostor.ba${link}`;
 

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -399,9 +399,7 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${
-              extractedData["re_realEstates_inserted"]
-            }`
+            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
           };
         }
 
@@ -412,9 +410,7 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${
-              extractedData["re_realEstates_edited"]
-            }`
+            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 
@@ -218,7 +218,7 @@ class SaljicCrawler {
   }
 
   async scrapeAd(url, adType) {
-    // console.log("[SALJIC] Scraping : ", url);
+    console.log("[SALJIC] Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
@@ -227,7 +227,9 @@ class SaljicCrawler {
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
+      const agencyObjectId = url
+        ? parseInt(url.substring(46, url.length))
+        : null;
 
       //Extracting main properties
       const propertySelectors = {
@@ -272,14 +274,10 @@ class SaljicCrawler {
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
+      const tmpLatLong = latAndLongSrc.split("marker=")[1];
+      const latText = tmpLatLong.split("%2C")[0];
+      const longText = tmpLatLong.split("%2C")[1];
+
       const locationLat = parseFloat(latText) || null;
       const locationLong = parseFloat(longText) || null;
 
@@ -328,11 +326,10 @@ class SaljicCrawler {
       let numberOfViewsKivi = null;
       let streetNumber = 0;
       let adStatus = status;
-      let shortDescription = descriptions.substring(
-        0,
-        descriptions.indexOf(".")
-      );
-      let longDescription = descriptions;
+      let shortDescription = descriptions
+        ? descriptions.substring(0, descriptions.indexOf("."))
+        : "";
+      let longDescription = descriptions || "";
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -343,10 +340,14 @@ class SaljicCrawler {
           .replace(/[\n\r\t]/gm, "")
           .trim();
 
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldTitle = mainField
+          ? mainField.substring(0, mainField.indexOf("  "))
+          : "";
         const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
+          ? mainField
+              .substring(mainField.indexOf("  "), mainField.length)
+              .trim()
+          : "";
 
         switch (mainFieldTitle) {
           case "Površina":

app/helpers/db/realEstate.js

@@ -332,14 +332,10 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where:
-      includeIncompleteAds || includeIncompleteAds == null
-        ? queryIncludeIncomplete
-        : query,
+    where: includeIncompleteAds ? queryIncludeIncomplete : query,
     limit: maxResults,
     order
   });

app/helpers/db/searchRequest.js

@@ -157,7 +157,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
+      //Or ones that picked some values but also picked to includeIncomplete ads
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -176,10 +176,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -229,10 +226,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -281,10 +275,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -322,10 +313,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -359,10 +347,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -396,10 +381,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -441,13 +423,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query which is default
+  //Tag to check if incomplete ads are accepted in query
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.or]: {
-        [Op.eq]: true,
-        [Op.is]: null
-      }
+      [Op.eq]: true
     };
   }
 

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,7 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { NO_CHECK_UP_DAYS } = require("../../config/appConfig");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -45,9 +45,9 @@ const findNotNotifiedMatches = async () => {
 };
 const findAllRequestsForCheckUp = async () => {
   //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate NO_CHECK_UP_DAYS days from now
+  //The ones that received notification for real estate CHECK_UP_DAYS days from now
   const date = new Date();
-  const checkUpDate = date.getDate() - NO_CHECK_UP_DAYS;
+  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
   date.setDate(checkUpDate);
   const dateQuery = {
     createdAt: {

app/helpers/emailContentGenerator.js

@@ -3,12 +3,12 @@
 const {
   MAX_REAL_ESTATES_IN_EMAIL,
   APP_URL,
-  USE_KIVI_ENVIRONMENT_TAG
+  STAGING
 } = require("../config/appConfig");
 const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-//Tag to recognize staging from development if needed
-const stagingTag = USE_KIVI_ENVIRONMENT_TAG ? "[STAGING] " : "";
+//Tag to recognize staging from development
+const stagingTag = STAGING ? "[STAGING] " : "";
 
 const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
   return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>

app/helpers/fetchWrapper.js

@@ -1,13 +0,0 @@
-const nodeFetch = require("node-fetch");
-const { USER_AGENT } = require("../config/appConfig");
-
-const fetch = async (url, options = {}) => {
-  const newOptions = Object.assign({}, options);
-  if (!newOptions["headers"]) {
-    newOptions["headers"] = {};
-  }
-  newOptions["headers"]["User-Agent"] = USER_AGENT;
-  return nodeFetch(url, newOptions);
-};
-
-module.exports = fetch;

app/models/searchRequest.js

@@ -15,7 +15,15 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+        coordinates: [
+          [
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0]
+          ]
+        ],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },

app/services/notificationService.js

@@ -1,7 +1,7 @@
 "use strict";
-const { USE_KIVI_ENVIRONMENT_TAG } = require("../config/appConfig");
+const { STAGING } = require("../config/appConfig");
 
-const stagingTag = USE_KIVI_ENVIRONMENT_TAG ? "[STAGING] " : "";
+const stagingTag = STAGING ? "[STAGING] " : "";
 
 const {
   matchRealEstates,
@@ -131,7 +131,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+ /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -144,7 +144,7 @@ const checkUpNotify = async () => {
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
   }
-  await Promise.all(asyncSendEmailActions); */
+  await Promise.all(asyncSendEmailActions); */ 
 };
 
 module.exports = {

app/views/advancedFilters.ejs

@@ -61,8 +61,9 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
+      <% if (includeIncompleteAds) { %>
            checked
-      >
+      <% } %>>
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

app/views/location.ejs

@@ -217,7 +217,7 @@
   });
 </script>
 <script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
+  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
   async
   defer
 ></script>

development.env

@@ -8,19 +8,18 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-KIVI_ENVIRONMENT=Variable to denote development, staging and production
+ENVIRONMENT=Variable to denote development, staging and production
 
-USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
 
-NO_CHECK_UP_DAYS=Check up email is sent after this number of days without notification
+CHECK_UP_DAYS=Check up email is sent after this number of days without notification
 #=============== GOOGLE ANALYTICS =============#
 GA_ID=Google Analytics ID
 
 #=============== GOOGLE MAPS =============#
-GOOGLE_MAP_KEY=(your-key-here)
+API_MAP_KEY=(your-key-here)
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
@@ -70,4 +69,4 @@ AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found