refactoring

app/services/crawlerService.js

@@ -5,42 +5,65 @@ const db = require("../models/index");
 const MarketAlert = require("../models/marketalert");
 
 const crawlers = [
-    new OlxCrawler(1, 2, 3),
-    // new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
-  ];
+  new OlxCrawler(1, 2, 3),
+  // new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
+];
 
-  async function crawlAll() {
-    console.log(db.MarketAlert);
+async function crawlAll() {
+  const properties = db.MarketAlert.rawAttributes;
+  console.log(properties);
 
-    for (let crawler of crawlers) {
-      try {
-        let results = await crawler.crawl();
+  for (let crawler of crawlers) {
+    try {
+      let results = await crawler.crawl();
+      const marketAlerts = [];
 
-        for (const result of results) {
+      for (const result of results) {
+        console.log("This is result", result);
+        console.log("This is result", result.size);
 
-            const newMAlert = Object.assign({}, MarketAlert)
-            console.log(newMAlert);
-            db.MarketAlert.bulkCreate( [{
+        // category: category,
+        // url,
+        // title,
+        // price: isNaN(parsedPrice) ? price : parsedPrice,
+        // size: parseFloat(size),
+        // rooms: parsedRooms,
+        // floor: parseInt(floor),
+        // address,
+        // location,
+        // // adType: AD_TYPE_SALE,
+        // time,
+        // shortDescription: descriptions.first().text(),
+        // longDescription: descriptions.last().text(),
+        // lat,
+        // lng,
+        // loc: [parseFloat(lat), parseFloat(lng)],
 
-            }])
-        }
-        // for (let saver of savers) {
-        //   try {
-        //     await saver.connect();
-        //     await saver.save(results);
-        //   } catch (e) {
-        //     console.log("Error saving. Trying next saver! ", e);
-        //   }
-        // }
-      } catch (e) {
-        console.log("Error crawling. Trying next crawler! ", e);
+        marketAlerts.push({
+          url: result.url,
+          realestateOrigin: "OLX",
+          originId: "1",
+          size: "" + result.size,
+          price: result.price,
+          email: "em"
+          // lastDate: DataTypes.STRING,
+          // municipailty: DataTypes.STRING,
+          // region: DataTypes.STRING,
+          // gardenSize: DataTypes.INTEGER,
+
+
+        })
       }
-    }
-   
-  
-    for (let saver of savers) {
-      saver.close();
+
+      try {
+        await db.MarketAlert.bulkCreate(marketAlerts);
+      } catch (e) {
+        console.log("Could not bulkCreate marketalers reason: ", e);
+      }
+    } catch (e) {
+      console.log("Error crawling. Trying next crawler! ", e);
     }
   }
-  
-  crawlAll();
+}
+
+crawlAll();