From 753a09aa36b67d894124a46498a7f2fef75d4705 Mon Sep 17 00:00:00 2001
From: Nedim Uka <nedim@saburly.com>
Date: Fri, 12 Jul 2019 16:13:03 +0200
Subject: [PATCH] Fixed crawler not reading and comparing all RERequest results

---
 app/controllers/realEstates.js     | 10 ++++++++++
 app/helpers/crawlers/olxClawler.js |  9 ++++++---
 app/services/crawlerService.js     |  4 +++-
 app/views/realEstates.ejs          | 16 ++++++++++++++++
 index.js                           | 11 +++++++++++
 5 files changed, 46 insertions(+), 4 deletions(-)
 create mode 100644 app/controllers/realEstates.js
 create mode 100644 app/views/realEstates.ejs

diff --git a/app/controllers/realEstates.js b/app/controllers/realEstates.js
new file mode 100644
index 0000000..6d3783d
--- /dev/null
+++ b/app/controllers/realEstates.js
@@ -0,0 +1,10 @@
+
+const getRealEstates = (req,res) => {
+    const title = "Ovo su nekretnine koje smo pronašli za vas"
+    res.render('realEstates', { nextStep: '/nekretnine', title } );
+  };
+  
+  module.exports = {
+    getRealEstates
+  };
+  
\ No newline at end of file
diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js
index e94daa7..42cb57f 100644
--- a/app/helpers/crawlers/olxClawler.js
+++ b/app/helpers/crawlers/olxClawler.js
@@ -36,10 +36,13 @@ module.exports = class OlxCrawler {
         const urls = this.createRequestUrls(realestateRequests);
         let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
         console.log("Final crawler results");
-        if (results[0]) {
-            console.log(results[0].length);
+        // console.log(results);
+        const flatResults = results.flat(); 
+        console.log(flatResults);
+        if (flatResults) {
+            console.log(flatResults.length);
 
-            for (const finalResult of results[0]) {
+            for (const finalResult of flatResults) {
 
                 if (null !== finalResult) {
                     if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js
index e410c4c..0c9e072 100644
--- a/app/services/crawlerService.js
+++ b/app/services/crawlerService.js
@@ -69,7 +69,9 @@ async function crawlAll() {
 
             try {
 
-                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => { 
+
+                    return (elem.url === url && elem.request === request) }));
                 console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
 
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);
diff --git a/app/views/realEstates.ejs b/app/views/realEstates.ejs
new file mode 100644
index 0000000..4e1a98e
--- /dev/null
+++ b/app/views/realEstates.ejs
@@ -0,0 +1,16 @@
+<!--suppress HtmlUnknownAnchorTarget -->
+<% include partials/navBar %> 
+
+<div class="row center-align">
+    <ul class="collection with-header">
+      <% for(const realEstate of realEstates) { %>
+      <li class="collection-item">
+        <div><%= realEstate.title %>
+          <a href="realEstate.url" class="secondary-content">
+            <i class="material-icons">send</i>
+          </a>
+        </div>
+      </li>
+      <% } %>
+    </ul>
+  </div>
\ No newline at end of file
diff --git a/index.js b/index.js
index 6718d83..c1a0b7e 100644
--- a/index.js
+++ b/index.js
@@ -167,3 +167,14 @@ var rule = new schedule.RecurrenceRule();
     await processNotifications();
     console.log(new Date(), 'Notification service finished');
   });
+
+ /**
+  * Add flat method to Array
+  */
+  Object.defineProperty(Array.prototype, 'flat', {
+    value: function(depth = 1) {
+      return this.reduce(function (flat, toFlatten) {
+        return flat.concat((Array.isArray(toFlatten) && (depth>1)) ? toFlatten.flat(depth-1) : toFlatten);
+      }, []);
+    }
+});