Fixed crawler not reading and comparing all RERequest results

app/controllers/realEstates.js

@@ -0,0 +1,10 @@
+
+const getRealEstates = (req,res) => {
+    const title = "Ovo su nekretnine koje smo pronašli za vas"
+    res.render('realEstates', { nextStep: '/nekretnine', title } );
+  };
+  
+  module.exports = {
+    getRealEstates
+  };
+  

app/helpers/crawlers/olxClawler.js

@@ -36,10 +36,13 @@ module.exports = class OlxCrawler {
         const urls = this.createRequestUrls(realestateRequests);
         let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
         console.log("Final crawler results");
-        if (results[0]) {
+        // console.log(results);
-            console.log(results[0].length);
+        const flatResults = results.flat(); 
+        console.log(flatResults);
+        if (flatResults) {
+            console.log(flatResults.length);
 
-            for (const finalResult of results[0]) {
+            for (const finalResult of flatResults) {
 
                 if (null !== finalResult) {
                     if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {

app/services/crawlerService.js

@@ -69,7 +69,9 @@ async function crawlAll() {
 
             try {
 
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => { 
+
+                    return (elem.url === url && elem.request === request) }));
                 console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
 
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);

app/views/realEstates.ejs

@@ -0,0 +1,16 @@
+<!--suppress HtmlUnknownAnchorTarget -->
+<% include partials/navBar %> 
+
+<div class="row center-align">
+    <ul class="collection with-header">
+      <% for(const realEstate of realEstates) { %>
+      <li class="collection-item">
+        <div><%= realEstate.title %>
+          <a href="realEstate.url" class="secondary-content">
+            <i class="material-icons">send</i>
+          </a>
+        </div>
+      </li>
+      <% } %>
+    </ul>
+  </div>

index.js

@@ -167,3 +167,14 @@ var rule = new schedule.RecurrenceRule();
     await processNotifications();
     console.log(new Date(), 'Notification service finished');
   });
+
+ /**
+  * Add flat method to Array
+  */
+  Object.defineProperty(Array.prototype, 'flat', {
+    value: function(depth = 1) {
+      return this.reduce(function (flat, toFlatten) {
+        return flat.concat((Array.isArray(toFlatten) && (depth>1)) ? toFlatten.flat(depth-1) : toFlatten);
+      }, []);
+    }
+});