Changed email content to show number of all matching real estates.

Map key env var

See merge request saburly/marketalarm/web!76

.gitignore

@@ -1,3 +1,5 @@
 node_modules/
 .env
 .idea/
+.eslintrc
+.vscode/

app/config/appConfig.js

@@ -30,6 +30,8 @@ const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
 
 const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
+const API_MAP_KEY = process.env.API_MAP_KEY || "";
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -39,5 +41,6 @@ module.exports = {
   AWS_EMAIL_CONFIG,
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
-  PRINT_CRAWLER_DEBUG
+  PRINT_CRAWLER_DEBUG,
+  API_MAP_KEY
 };

app/crawler/specificCrawlers/rental.js

@@ -312,7 +312,7 @@ class RentalCrawler {
 
         let numberOfRooms =
             parseInt(extractedData["re_realEstates_roomsNO"]) +
-              parseInt(extractedData["re_realEstates_bedroomNO"]) || null,
+              parseInt(extractedData["re_realEstates_bedNO"]) || null,
           numberOfFloors =
             parseInt(extractedData["re_realEstates_floorsNO"]) ||
             this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]),
@@ -352,7 +352,9 @@ class RentalCrawler {
             realEstatePropertiesFromInfrastructure.phoneConnection,
           cableTV = realEstatePropertiesFromInfrastructure.cableTV,
           internet = realEstatePropertiesFromInfrastructure.internet,
-          basementAttic = realEstatePropertiesFromSpaces.basementAttic,
+          basementAttic =
+            realEstatePropertiesFromSpaces.basementAttic ||
+            this.checkBasemAtticFromFloors(extractedData["re_floorNO_id"]),
           storeRoom = realEstatePropertiesFromSpaces.storeRoom,
           videoSurveillance =
             realEstatePropertiesFromDescriptions.videoSurveillance ||
@@ -397,9 +399,7 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${
+            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
-              extractedData["re_realEstates_inserted"]
-            }`
           };
         }
 
@@ -410,9 +410,7 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${
+            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
-              extractedData["re_realEstates_edited"]
-            }`
           };
         }
 
@@ -782,8 +780,42 @@ class RentalCrawler {
     if (floorIds.length === 0) {
       return null;
     }
+    let noOfFloors = floorIds.length;
+    // Floors of 'suteren', 'podrum', 'tavan' and 'potkrovlje' are not counted
+    floorIds.forEach(id => {
+      if (
+        parseInt(id) === 1 ||
+        parseInt(id) === 2 ||
+        parseInt(id) === 12 ||
+        parseInt(id) === 14
+      ) {
+        noOfFloors--;
+      }
+    });
+    return noOfFloors;
+  }
 
-    return floorIds.length;
+  checkBasemAtticFromFloors(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+    const floorIds = floorsIdText.split(",");
+
+    let check = false;
+
+    if (floorIds.length === 0) {
+      check = false;
+    }
+    //If floors 'suteren', 'podrum', 'tavan' and 'potkrovlje' exists then tag for basement-attic is true
+    floorIds.forEach(id => {
+      if (
+        parseInt(id) === 1 ||
+        parseInt(id) === 2 ||
+        parseInt(id) === 12 ||
+        parseInt(id) === 14
+      ) {
+        check = true;
+      }
+    });
+    return check;
   }
 
   async sleep(ms) {

app/helpers/emailContentGenerator.js

@@ -23,16 +23,19 @@ const generateRealEstateLinks = realEstates => {
 const generateNotificationEmail = (
   realEstates,
   searchRequestId,
+  noAllRealEstates,
   dailyNotification = false
 ) => {
   const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
+
   const realEstatesToShow = truncateList
     ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
     : realEstates;
 
   const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
+
   const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
-  const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
+  const moreRealEstates = `<div>Kompletan spisak nekretnina (${noAllRealEstates}) možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
   const emailFooter = generateEmailFooter(searchRequestId);
   const asapMessageBody =
     realEstates.length > 1
@@ -70,6 +73,7 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
   } = searchRequest;
 
   const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
+
   const instantRealEstatesText = `<br/>
        <div>
         U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>

app/services/notificationService.js

@@ -8,7 +8,10 @@ const {
   generateNewSearchRequestEmail,
   generateEmailSubject
 } = require("../helpers/emailContentGenerator");
-const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
+const {
+  findNotNotifiedMatches,
+  findRealEstatesForSearchRequest
+} = require("../helpers/db/searchRequestMatch");
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -39,10 +42,18 @@ const notifyMatches = async (matches, dailyNotification = false) => {
     const { email, subscribed } = searchRequest;
     if (notifyNow && subscribed) {
       const allMatchingRealEstates = matches[id].realEstates || [];
+
+      //Variable allMatchingRealEstates are real estates that are "new" on the market
+      //the ones that we notify user in this moment, not all that already exists in db
+      //New variable allRealEstates are all real estates that exists in db for search req
+      const allRealEstates = await findRealEstatesForSearchRequest(id);
+      const noAllRealEstates = allRealEstates.length;
+
       if (allMatchingRealEstates.length > 0) {
         const emailContent = generateNotificationEmail(
           allMatchingRealEstates,
           id,
+          noAllRealEstates,
           dailyNotification
         );
         const emailSubject = generateEmailSubject(

app/views/location.ejs

@@ -177,5 +177,5 @@
     });
   });
 </script>
-<script src="https://maps.googleapis.com/maps/api/js?key=AIzaSyAna8ohfV2HBMcxGk_29vqxU5Z_bDickqg&language=bs&libraries=places&callback=initMap" async
+<script src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap" async
   defer></script>

development.env

@@ -14,6 +14,9 @@ MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in
 #=============== GOOGLE ANALYTICS =============#
 GA_ID=Google Analytics ID
 
+#=============== GOOGLE MAPS =============#
+API_MAP_KEY=(your-key-here)
+
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)

test/rentalScrapeTest.js

@@ -13,5 +13,5 @@ if (urlToScrape) {
   })();
 } else {
   console.log("No URL to scrape. Use like this : ");
-  console.log("npm run test-olx-scraper -- URL_TO_SCRAPE");
+  console.log("npm run test-rental-scraper -- URL_TO_SCRAPE");
 }