replace old crawler, without specific crawler and saver implementation
This commit is contained in:
@@ -1,77 +0,0 @@
|
||||
const Promise = require("bluebird");
|
||||
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
||||
const db = require("../models/index");
|
||||
const { allMarketAlerts } = require("../helpers/db/dbHelper");
|
||||
|
||||
async function crawlAll() {
|
||||
try {
|
||||
const marketAlertsFromDb = await allMarketAlerts(true);
|
||||
const hrefs = [];
|
||||
|
||||
marketAlertsFromDb.map(marketAlert => {
|
||||
if (hrefs[marketAlert.request] === undefined) {
|
||||
hrefs[marketAlert.request] = [];
|
||||
}
|
||||
|
||||
hrefs[marketAlert.request].push(marketAlert.url);
|
||||
});
|
||||
|
||||
const olxCrawler = new OlxCrawler(hrefs);
|
||||
|
||||
const crawlers = [olxCrawler];
|
||||
|
||||
return Promise.map(crawlers, function(crawler) {
|
||||
return crawler.crawl();
|
||||
}).then(async results => {
|
||||
try {
|
||||
const marketAlertsFromDb = await allMarketAlerts(false, true);
|
||||
|
||||
const marketAlerts = [];
|
||||
const mergedResults = [].concat.apply([], results);
|
||||
|
||||
for (const result of mergedResults) {
|
||||
marketAlerts.push({
|
||||
url: result.url,
|
||||
realestateOrigin: "OLX",
|
||||
originId: 1,
|
||||
size: result.size,
|
||||
price: result.price,
|
||||
email: result.email,
|
||||
request: result.uuid,
|
||||
municipality: result.municipality,
|
||||
region: result.region,
|
||||
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
|
||||
realEstateType: result.realEstateType,
|
||||
title: result.title,
|
||||
notified: false,
|
||||
hasLocation: result.hasLocation
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const filteredMarketAlerts = marketAlerts.filter(
|
||||
elem =>
|
||||
!marketAlertsFromDb.find(({ url, request }) => {
|
||||
return elem.url === url && elem.request === request;
|
||||
})
|
||||
);
|
||||
|
||||
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
|
||||
} catch (e) {
|
||||
console.log(
|
||||
"CRAWLER SERVICE: Could not bulkCreate marketalers reason: ",
|
||||
e
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(
|
||||
"CRAWLER SERVICE: Error crawling. Trying next crawler! ",
|
||||
e
|
||||
);
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
|
||||
}
|
||||
}
|
||||
module.exports = crawlAll;
|
||||
Reference in New Issue
Block a user