From 1bcc5e8e5d31541e703998d725317f16912de029 Mon Sep 17 00:00:00 2001
From: Nedim Uka <nedim@saburly.com>
Date: Thu, 20 Jun 2019 14:51:14 +0200
Subject: [PATCH] Preparing to save results to db

---
 app/helpers/crawlers/olxClawler.js |  6 ++----
 app/helpers/db/dbHelper.js         | 10 ++++++++++
 app/helpers/url.js                 |  2 +-
 app/models/marketalert.js          |  6 ++++++
 app/services/crawlerService.js     | 29 +++++++++++++++++++++--------
 5 files changed, 40 insertions(+), 13 deletions(-)
 create mode 100644 app/helpers/db/dbHelper.js

diff --git a/app/helpers/crawlers/olxClawler.js b/app/helpers/crawlers/olxClawler.js
index 3584bf6..075cffe 100644
--- a/app/helpers/crawlers/olxClawler.js
+++ b/app/helpers/crawlers/olxClawler.js
@@ -110,8 +110,6 @@ module.exports = class OlxCrawler {
         return null;
     }
 
-
-
     async indexPage(url, maxResults = 1000) {
         try {
             // console.log('Starting to index page: ' + pageNr);
@@ -138,7 +136,7 @@ module.exports = class OlxCrawler {
                 if (singleData) {
                     results.push(singleData);
                 }
-                await this.sleep(500);
+                // await this.sleep(500);
             }
 
             return results;
@@ -177,7 +175,7 @@ module.exports = class OlxCrawler {
             let result = await this.indexPage(url, maxResults);
             // Object.assign(results, result)
             results.push(result);
-            await this.sleep(5000);
+            // await this.sleep(5000);
         }
         return results;
     }
diff --git a/app/helpers/db/dbHelper.js b/app/helpers/db/dbHelper.js
new file mode 100644
index 0000000..6c81004
--- /dev/null
+++ b/app/helpers/db/dbHelper.js
@@ -0,0 +1,10 @@
+
+// const db = require('../../models/index');
+
+
+// const bulkInsert = async (reuslts) => {
+//     db.MarketAlert.bulkCreate({
+
+//     })
+   
+//   }
\ No newline at end of file
diff --git a/app/helpers/url.js b/app/helpers/url.js
index d36e04a..854d2a7 100644
--- a/app/helpers/url.js
+++ b/app/helpers/url.js
@@ -7,7 +7,7 @@ const currentRERequest = async (req) => {
   const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
   return request;
 };
-
+// TODO Fetch only subscribed realestate requests
 const allRERequest = async () => {
   return await db.RealEstateRequest.findAll();
 }
diff --git a/app/models/marketalert.js b/app/models/marketalert.js
index c0665a3..3aa2d42 100644
--- a/app/models/marketalert.js
+++ b/app/models/marketalert.js
@@ -5,6 +5,12 @@ module.exports = (sequelize, DataTypes) => {
     realestateOrigin: DataTypes.STRING,
     originId: DataTypes.STRING,
     lastDate: DataTypes.STRING,
+    size : DataTypes.INTEGER,
+    gardenSize : DataTypes.INTEGER,
+    price : DataTypes.INTEGER,
+    municipailty : DataTypes.STRING,
+    region : DataTypes.STRING,
+    
     email: { 
       type: DataTypes.STRING,
       allowNul: false
diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js
index d211f1f..13a7f3b 100644
--- a/app/services/crawlerService.js
+++ b/app/services/crawlerService.js
@@ -1,6 +1,8 @@
 
 // import OlxCrawler from '../helpers/crawlers/olixClawler'
 const OlxCrawler = require("../helpers/crawlers/olxClawler");
+const db = require("../models/index");
+const MarketAlert = require("../models/marketalert");
 
 const crawlers = [
     new OlxCrawler(1, 2, 3),
@@ -8,22 +10,33 @@ const crawlers = [
   ];
 
   async function crawlAll() {
+    console.log(db.MarketAlert);
 
     for (let crawler of crawlers) {
       try {
-        let results = await crawler.crawl()
-        for (let saver of savers) {
-          try {
-            await saver.connect();
-            await saver.save(results);
-          } catch (e) {
-            console.log("Error saving. Trying next saver! ", e);
-          }
+        let results = await crawler.crawl();
+
+        for (const result of results) {
+
+            const newMAlert = Object.assign({}, MarketAlert)
+            console.log(newMAlert);
+            db.MarketAlert.bulkCreate( [{
+
+            }])
         }
+        // for (let saver of savers) {
+        //   try {
+        //     await saver.connect();
+        //     await saver.save(results);
+        //   } catch (e) {
+        //     console.log("Error saving. Trying next saver! ", e);
+        //   }
+        // }
       } catch (e) {
         console.log("Error crawling. Trying next crawler! ", e);
       }
     }
+   
   
     for (let saver of savers) {
       saver.close();