Filter data by geolocation now sets hasLocation boolean instead of excluding results

app/helpers/crawlers/olxClawler.js

@@ -45,7 +45,12 @@ module.exports = class OlxCrawler {
                     if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
                         const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
 
+                    
                         if (pointInsideBoundingBox[0].length !== 0) {
+                            finalResult.hasLocation = true
+                            filteredResults.push(finalResult);
+                        } else {
+                            finalResult.hasLocation = false
                             filteredResults.push(finalResult);
                         }
                     }
@@ -270,6 +275,16 @@ class Indexer {
             if (this.olxUrl.url === undefined) {
                 return {}
             }
+
+            if (global.hrefs) {
+
+                if (global.hrefs[this.olxUrl.uuid] && global.hrefs[this.olxUrl.uuid].includes(this.olxUrl.url)) {
+
+                    console.log("We found duplicate URL");
+                    return null
+                }
+            }
+
             const res = await fetch(this.olxUrl.url);
             const body = await res.text();
             const $ = cheerio.load(body);

app/helpers/db/dbHelper.js

@@ -26,7 +26,7 @@ const allRERequestByUiid = async (requestArray) => {
 }
 
 /**
- * Find all , or all depending on notified bolean marketalerts, and order them by email
+ * Find all , or all depending on notified bolean marketalerts, that the hasLocation is true, and order them by email
  * 
  * @param fechAll bolean
  * @param notified bolean
@@ -43,7 +43,8 @@ const allMarketAlerts = async (fetchAll, notified) => {
 
     if (!fetchAll){
         queryObject.where = {
-            notified: notified
+            notified: notified,
+            hasLocation: true
         }
     }