Preparing to save results to db

app/helpers/crawlers/olxClawler.js

@@ -110,8 +110,6 @@ module.exports = class OlxCrawler {
         return null;
     }
 
-
-
     async indexPage(url, maxResults = 1000) {
         try {
             // console.log('Starting to index page: ' + pageNr);
@@ -138,7 +136,7 @@ module.exports = class OlxCrawler {
                 if (singleData) {
                     results.push(singleData);
                 }
-                await this.sleep(500);
+                // await this.sleep(500);
             }
 
             return results;
@@ -177,7 +175,7 @@ module.exports = class OlxCrawler {
             let result = await this.indexPage(url, maxResults);
             // Object.assign(results, result)
             results.push(result);
-            await this.sleep(5000);
+            // await this.sleep(5000);
         }
         return results;
     }

app/helpers/db/dbHelper.js

@@ -0,0 +1,10 @@
+
+// const db = require('../../models/index');
+
+
+// const bulkInsert = async (reuslts) => {
+//     db.MarketAlert.bulkCreate({
+
+//     })
+   
+//   }

app/helpers/url.js

@@ -7,7 +7,7 @@ const currentRERequest = async (req) => {
   const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
   return request;
 };
-
+// TODO Fetch only subscribed realestate requests
 const allRERequest = async () => {
   return await db.RealEstateRequest.findAll();
 }

app/models/marketalert.js

@@ -5,6 +5,12 @@ module.exports = (sequelize, DataTypes) => {
     realestateOrigin: DataTypes.STRING,
     originId: DataTypes.STRING,
     lastDate: DataTypes.STRING,
+    size : DataTypes.INTEGER,
+    gardenSize : DataTypes.INTEGER,
+    price : DataTypes.INTEGER,
+    municipailty : DataTypes.STRING,
+    region : DataTypes.STRING,
+    
     email: { 
       type: DataTypes.STRING,
       allowNul: false

app/services/crawlerService.js

@@ -1,6 +1,8 @@
 
 // import OlxCrawler from '../helpers/crawlers/olixClawler'
 const OlxCrawler = require("../helpers/crawlers/olxClawler");
+const db = require("../models/index");
+const MarketAlert = require("../models/marketalert");
 
 const crawlers = [
     new OlxCrawler(1, 2, 3),
@@ -8,22 +10,33 @@ const crawlers = [
   ];
 
   async function crawlAll() {
+    console.log(db.MarketAlert);
 
     for (let crawler of crawlers) {
       try {
-        let results = await crawler.crawl()
-        for (let saver of savers) {
-          try {
-            await saver.connect();
-            await saver.save(results);
-          } catch (e) {
-            console.log("Error saving. Trying next saver! ", e);
-          }
+        let results = await crawler.crawl();
+
+        for (const result of results) {
+
+            const newMAlert = Object.assign({}, MarketAlert)
+            console.log(newMAlert);
+            db.MarketAlert.bulkCreate( [{
+
+            }])
         }
+        // for (let saver of savers) {
+        //   try {
+        //     await saver.connect();
+        //     await saver.save(results);
+        //   } catch (e) {
+        //     console.log("Error saving. Trying next saver! ", e);
+        //   }
+        // }
       } catch (e) {
         console.log("Error crawling. Trying next crawler! ", e);
       }
     }
+   
   
     for (let saver of savers) {
       saver.close();