senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added realestate link to bulk email

Browse Source
This commit is contained in:
Nedim Uka
2019-07-12 18:00:02 +02:00
parent 753a09aa36
commit 81c30c36ec
7 changed files with 116 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
app/services/crawlerService.js
View File

@@ -4,11 +4,6 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');
const olxCrawler = new OlxCrawler(1, 2, 3);
const crawlers = [
olxCrawler,
];
async function crawlAll() {
console.log("CRAWLER SERVICE: crawlAll");
@@ -16,7 +11,7 @@ async function crawlAll() {
try {
const marketAlertsFromDb = await allMarketAlerts(true);
const hrefs = [];
marketAlertsFromDb.map(marketAlert => {
if (hrefs[marketAlert.request] === undefined) {
hrefs[marketAlert.request] = []
@@ -25,64 +20,66 @@ async function crawlAll() {
hrefs[marketAlert.request].push(marketAlert.url);
})
global.hrefs = hrefs;
console.log("CRAWLER SERVICE: GLOBAL HREFS");
console.log(global.hrefs);
console.log(hrefs);
const olxCrawler = new OlxCrawler(hrefs);
} catch (e) {
console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
}
return Promise.map(crawlers, function (crawler) {
return crawler.crawl();
}).then(async (results) => {
try {
const marketAlertsFromDb = await allMarketAlerts(false, true);
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
for (const result of mergedResults) {
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: 1,
size: result.size,
price: result.price,
email: result.email,
request: result.uuid,
// lastDate: DataTypes.STRING,
municipality: result.municipality,
region: result.region,
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
realEstateType: result.realEstateType,
title: result.title,
notified: false,
hasLocation: result.hasLocation
})
}
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
const crawlers = [
olxCrawler,
];
return Promise.map(crawlers, function (crawler) {
return crawler.crawl();
}).then(async (results) => {
try {
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => {
const marketAlertsFromDb = await allMarketAlerts(false, true);
return (elem.url === url && elem.request === request) }));
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
for (const result of mergedResults) {
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: 1,
size: result.size,
price: result.price,
email: result.email,
request: result.uuid,
municipality: result.municipality,
region: result.region,
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
realEstateType: result.realEstateType,
title: result.title,
notified: false,
hasLocation: result.hasLocation
})
}
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
try {
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url, request }) => {
return (elem.url === url && elem.request === request)
}));
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
} catch (e) {
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
}
} catch (e) {
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
}
} catch (e) {
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
}
})
})
} catch (e) {
console.error("CRAWLER SERVICE:could not fetch marketalerts ", e);
}
};
module.exports = crawlAll;
// crawlAll();