old-web/app/services/crawlerService.js

const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");

const olxCrawler = new OlxCrawler(1, 2, 3);

const crawlers = [
    olxCrawler,
];

async function crawlAll() {

    Promise.map(crawlers, function (crawler) {
        return crawler.crawl();
    }).then(async (results) => {

        try {

            const marketAlertsFromDb = await db.MarketAlert.findAll();

            const marketAlerts = [];
            const mergedResults = [].concat.apply([], results);

            for (const result of mergedResults) {
                marketAlerts.push({
                    url: result.url,
                    realestateOrigin: "OLX",
                    originId: 1,
                    size: result.size,
                    price: result.price,
                    email: result.email,
                    // lastDate: DataTypes.STRING,
                    municipality: result.municipality,
                    region: result.region,
                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
                    realEstateType: result.realEstateType
                })
            }
            try {
                console.log(marketAlerts);
                const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
                process.exit()
            } catch (e) {
                console.log("Could not bulkCreate marketalers reason: ", e);
            }
        } catch (e) {
            console.log("Error crawling. Trying next crawler! ", e);
        }
    })
};

crawlAll();