Filter data by geolocation now sets hasLocation boolean instead of excluding results

app/helpers/crawlers/olxClawler.js

@@ -45,7 +45,12 @@ module.exports = class OlxCrawler {
                     if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
                         const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
 
+                    
                         if (pointInsideBoundingBox[0].length !== 0) {
+                            finalResult.hasLocation = true
+                            filteredResults.push(finalResult);
+                        } else {
+                            finalResult.hasLocation = false
                             filteredResults.push(finalResult);
                         }
                     }
@@ -270,6 +275,16 @@ class Indexer {
             if (this.olxUrl.url === undefined) {
                 return {}
             }
+
+            if (global.hrefs) {
+
+                if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
+
+                    console.log("We found duplicate URL");
+                    return null
+                }
+            }
+
             const res = await fetch(this.olxUrl.url);
             const body = await res.text();
             const $ = cheerio.load(body);

app/helpers/db/dbHelper.js

@@ -26,7 +26,7 @@ const allRERequestByUiid = async (requestArray) => {
 }
 
 /**
- * Find all , or all depending on notified bolean marketalerts, and order them by email
+ * Find all , or all depending on notified bolean marketalerts, that the hasLocation is true, and order them by email
  * 
  * @param fechAll bolean
  * @param notified bolean
@@ -43,7 +43,8 @@ const allMarketAlerts = async (fetchAll, notified) => {
 
     if (!fetchAll){
         queryObject.where = {
-            notified: notified
+            notified: notified,
+            hasLocation: true
         }
     }
 

app/migrations/20190710141356-add-has-location-to-marketalerts.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'hasLocation',
+        {
+          type: Sequelize.BOOLEAN
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'hasLocation'
+    );
+  }
+};

app/models/marketalert.js

@@ -14,6 +14,7 @@ module.exports = (sequelize, DataTypes) => {
     notified : DataTypes.BOOLEAN,
     title : DataTypes.STRING,
     request: DataTypes.STRING,
+    hasLocation: DataTypes.BOOLEAN,
     
     email: { 
       type: DataTypes.STRING,

app/services/crawlerService.js

@@ -13,29 +13,35 @@ const crawlers = [
 async function crawlAll() {
     console.log("CRAWLER SERVICE: crawlAll");
 
+    try {
+        const marketAlertsFromDb = await allMarketAlerts(true);
+        const hrefs = [];
+       
+        marketAlertsFromDb.map(marketAlert => {
+            if (hrefs[marketAlert.request] === undefined) {
+                hrefs[marketAlert.request] = []
+            }
+
+            hrefs[marketAlert.request].push(marketAlert.url);
+        })
+
+        global.hrefs = hrefs;
+        console.log("CRAWLER SERVICE: GLOBAL HREFS");
+        console.log(global.hrefs);
+
+    } catch (e) {
+        console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
+    }
+
     return Promise.map(crawlers, function (crawler) {
         return crawler.crawl();
     }).then(async (results) => {
 
         try {
 
-            const marketAlertsFromDb = await allMarketAlerts(true);
-            const hrefs = [];
-            const subscribedMakretAlerts = marketAlertsFromDb.filter(marketAlert => {
-                return marketAlert.subscribed;
-            });
-            marketAlertsFromDb.map(marketAlert => {
-                if (hrefs[marketAlert.request] === undefined) {
-                    hrefs[marketAlert.request] = [] 
-                }
+            const marketAlertsFromDb = await allMarketAlerts(false, true);
 
-                hrefs[marketAlert.request].push(marketAlert.url);
-            })
-
-            global.hrefs = hrefs;
-            console.log(global.hrefs);
-            
-            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + subscribedMakretAlerts.length);
+            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
 
             const marketAlerts = [];
             const mergedResults = [].concat.apply([], results);
@@ -55,14 +61,15 @@ async function crawlAll() {
                     gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
                     realEstateType: result.realEstateType,
                     title: result.title,
-                    notified: false
+                    notified: false,
+                    hasLocation: result.hasLocation
                 })
             }
             console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
 
             try {
 
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !subscribedMakretAlerts.find(({ url }) => elem.url === url));
+                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
                 console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
 
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);