Merge branch 'master' after user-agent change into email-density

app/config/appConfig.js

@@ -45,9 +45,6 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
-const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
-const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
-
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -62,7 +59,5 @@ module.exports = {
   STAGING,
   CHECK_UP_DAYS,
   PROSTOR_LOGIN,
-  USER_AGENT,
+  USER_AGENT
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY
 };

app/helpers/db/searchRequest.js

@@ -156,33 +156,11 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       // If real estate dont have defined number of rooms ex. null
-      //It returns requests that didn't choose number of rooms - also null
+      //It returns all search requests except for ones that dont want incpomlete ads
-      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            [Op.and]: [
+        }
-              {
-                numberOfRoomsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfRoomsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
       };
     }
   }
@@ -212,30 +190,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       numberOfFloorsQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            [Op.and]: [
+        }
-              {
-                numberOfFloorsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfFloorsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
       };
     }
   }
@@ -264,30 +221,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else {
       floorQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            [Op.and]: [
+        }
-              {
-                floorMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                floorMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
       };
     }
   }
@@ -296,7 +232,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
   //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
   //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
   //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
-  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
+  //If real estate characteristic = null, dont know if true or false, find all req except ones that dont want incomplete ads
   let balconyQuery = {};
   if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
     if (balcony === false) {
@@ -307,30 +243,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (balcony === null) {
       balconyQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            balcony: {
+        }
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                balcony: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
       };
     }
   }
@@ -344,30 +259,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (newBuilding === null) {
       newBuildingQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            newBuilding: {
+        }
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                newBuilding: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
       };
     }
   }
@@ -381,33 +275,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
       };
     } else if (elevator === null) {
       elevatorQuery = {
-        [Op.or]: [
+        includeIncompleteAds: {
-          {
+          [Op.ne]: false
-            elevator: {
+        }
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                elevator: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
       };
     }
   }
+
   //General query consists of each individual query
   const query = {
     adType,

app/helpers/fetchWrapper.js

@@ -1,9 +1,5 @@
 const nodeFetch = require("node-fetch");
-const {
+const { USER_AGENT } = require("../config/appConfig");
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY
-} = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
   const newOptions = Object.assign({}, options);
@@ -11,11 +7,7 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
-  const urlAdaptedForScraping = USE_SCRAPER_API
+  return nodeFetch(url, newOptions);
-    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
-    : url;
-
-  return nodeFetch(urlAdaptedForScraping, newOptions);
 };
 
 module.exports = fetch;

development.env

@@ -22,10 +22,6 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
-#=============== SCRAPER API SUPORT =============#
-USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api 
-
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)