old-web/app/services/crawlerService.js

// import OlxCrawler from '../helpers/crawlers/olixClawler'
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const MarketAlert = require("../models/marketalert");

const crawlers = [
  new OlxCrawler(1, 2, 3),
  // new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
];

async function crawlAll() {
  const properties = db.MarketAlert.rawAttributes;
  console.log(properties);

  for (let crawler of crawlers) {
    try {
      let results = await crawler.crawl();
      const marketAlerts = [];

      for (const result of results) {
        console.log("This is result", result);
        console.log("This is result", result.size);

        // category: category,
        // url,
        // title,
        // price: isNaN(parsedPrice) ? price : parsedPrice,
        // size: parseFloat(size),
        // rooms: parsedRooms,
        // floor: parseInt(floor),
        // address,
        // location,
        // // adType: AD_TYPE_SALE,
        // time,
        // shortDescription: descriptions.first().text(),
        // longDescription: descriptions.last().text(),
        // lat,
        // lng,
        // loc: [parseFloat(lat), parseFloat(lng)],

        marketAlerts.push({
          url: result.url,
          realestateOrigin: "OLX",
          originId: "1",
          size: "" + result.size,
          price: result.price,
          email: "em"
          // lastDate: DataTypes.STRING,
          // municipailty: DataTypes.STRING,
          // region: DataTypes.STRING,
          // gardenSize: DataTypes.INTEGER,


        })
      }

      try {
        await db.MarketAlert.bulkCreate(marketAlerts);
      } catch (e) {
        console.log("Could not bulkCreate marketalers reason: ", e);
      }
    } catch (e) {
      console.log("Error crawling. Trying next crawler! ", e);
    }
  }
}

crawlAll();