59 lines
1.8 KiB
JavaScript
59 lines
1.8 KiB
JavaScript
|
|
const Promise = require("bluebird");
|
|
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
|
const db = require("../models/index");
|
|
|
|
const olxCrawler = new OlxCrawler(1, 2, 3);
|
|
const olxCrawler1 = new OlxCrawler(1, 2, 3);
|
|
const olxCrawler2 = new OlxCrawler(1, 2, 3);
|
|
|
|
const crawlers = [
|
|
olxCrawler,
|
|
olxCrawler1,
|
|
olxCrawler2,
|
|
];
|
|
|
|
async function crawlAll() {
|
|
|
|
Promise.map(crawlers, function (crawler) {
|
|
return crawler.crawl();
|
|
}).then(async (results) => {
|
|
|
|
try {
|
|
|
|
const marketAlertsFromDb = await db.MarketAlert.findAll();
|
|
|
|
const marketAlerts = [];
|
|
const mergedResults = [].concat.apply([], results);
|
|
|
|
for (const result of mergedResults) {
|
|
marketAlerts.push({
|
|
url: result.url,
|
|
realestateOrigin: "OLX",
|
|
originId: 1,
|
|
size: result.size,
|
|
price: result.price,
|
|
email: result.email,
|
|
// lastDate: DataTypes.STRING,
|
|
municipality: result.municipality,
|
|
region: result.region,
|
|
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
|
|
realEstateType: result.realEstateType
|
|
})
|
|
}
|
|
try {
|
|
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
|
|
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
|
|
process.exit()
|
|
} catch (e) {
|
|
console.log("Could not bulkCreate marketalers reason: ", e);
|
|
}
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
})
|
|
};
|
|
|
|
crawlAll();
|
|
|