old-web/app/services/crawlerService.js

const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');

const olxCrawler = new OlxCrawler(1, 2, 3);

const crawlers = [
    olxCrawler,
];

async function crawlAll() {
    console.log("CRAWLER SERVICE: crawlAll");

    Promise.map(crawlers, function (crawler) {
        return crawler.crawl();
    }).then(async (results) => {

        try {

            const marketAlertsFromDb = await allMarketAlerts();
            console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);

            const marketAlerts = [];
            const mergedResults = [].concat.apply([], results);

            for (const result of mergedResults) {
                marketAlerts.push({
                    url: result.url,
                    realestateOrigin: "OLX",
                    originId: 1,
                    size: result.size,
                    price: result.price,
                    email: result.email,
                    // lastDate: DataTypes.STRING,
                    municipality: result.municipality,
                    region: result.region,
                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
                    realEstateType: result.realEstateType
                })
            }
            console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);

            try {

                const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
                console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);

                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
                process.exit();

            } catch (e) {
                console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
                process.exit();
            }
        } catch (e) {
            console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
            process.exit();
        }
    })
};

crawlAll();