80 lines
2.8 KiB
JavaScript
80 lines
2.8 KiB
JavaScript
|
|
const Promise = require("bluebird");
|
|
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
|
const db = require("../models/index");
|
|
const { allMarketAlerts } = require('../helpers/db/dbHelper');
|
|
|
|
const olxCrawler = new OlxCrawler(1, 2, 3);
|
|
|
|
const crawlers = [
|
|
olxCrawler,
|
|
];
|
|
|
|
async function crawlAll() {
|
|
console.log("CRAWLER SERVICE: crawlAll");
|
|
|
|
return Promise.map(crawlers, function (crawler) {
|
|
return crawler.crawl();
|
|
}).then(async (results) => {
|
|
|
|
try {
|
|
|
|
const marketAlertsFromDb = await allMarketAlerts(true);
|
|
const hrefs = [];
|
|
const subscribedMakretAlerts = marketAlertsFromDb.filter(marketAlert => {
|
|
return marketAlert.subscribed;
|
|
});
|
|
marketAlertsFromDb.map(marketAlert => {
|
|
if (hrefs[marketAlert.request] === undefined) {
|
|
hrefs[marketAlert.request] = []
|
|
}
|
|
|
|
hrefs[marketAlert.request].push(marketAlert.url);
|
|
})
|
|
|
|
global.hrefs = hrefs;
|
|
console.log(global.hrefs);
|
|
|
|
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + subscribedMakretAlerts.length);
|
|
|
|
const marketAlerts = [];
|
|
const mergedResults = [].concat.apply([], results);
|
|
|
|
for (const result of mergedResults) {
|
|
marketAlerts.push({
|
|
url: result.url,
|
|
realestateOrigin: "OLX",
|
|
originId: 1,
|
|
size: result.size,
|
|
price: result.price,
|
|
email: result.email,
|
|
request: result.uuid,
|
|
// lastDate: DataTypes.STRING,
|
|
municipality: result.municipality,
|
|
region: result.region,
|
|
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
|
|
realEstateType: result.realEstateType,
|
|
title: result.title,
|
|
notified: false
|
|
})
|
|
}
|
|
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
|
|
|
|
try {
|
|
|
|
const filteredMarketAlerts = marketAlerts.filter((elem) => !subscribedMakretAlerts.find(({ url }) => elem.url === url));
|
|
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
|
|
|
|
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
|
|
|
|
} catch (e) {
|
|
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
|
|
}
|
|
} catch (e) {
|
|
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
|
|
}
|
|
})
|
|
};
|
|
module.exports = crawlAll;
|
|
// crawlAll();
|