senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added send notification service, and queried unsent marketalerts, fixed some issues with crawler, and added proper logging

Browse Source
This commit is contained in:
Nedim Uka
2019-06-25 17:06:07 +02:00
parent 5ffdaef1bf
commit 208faa08df
8 changed files with 123 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
app/services/crawlerService.js
View File

@@ -2,6 +2,7 @@
const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const { allMarketAlerts } = require('../helpers/db/dbHelper');
const olxCrawler = new OlxCrawler(1, 2, 3);
@@ -10,6 +11,7 @@ const crawlers = [
];
async function crawlAll() {
console.log("CRAWLER SERVICE: crawlAll");
Promise.map(crawlers, function (crawler) {
return crawler.crawl();
@@ -17,7 +19,8 @@ async function crawlAll() {
try {
const marketAlertsFromDb = await db.MarketAlert.findAll();
const marketAlertsFromDb = await allMarketAlerts();
console.log("CRAWLER SERVICE: number of existing MarketAlerts from db: " + marketAlertsFromDb.length);
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
@@ -37,16 +40,23 @@ async function crawlAll() {
realEstateType: result.realEstateType
})
}
console.log("CRAWLER SERVICE: Number of crawler results: " + marketAlerts.length);
try {
console.log(marketAlerts);
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
console.log("CRAWLER SERVICE: Number of new crawler results: " + filteredMarketAlerts.length);
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
process.exit()
process.exit();
} catch (e) {
console.log("Could not bulkCreate marketalers reason: ", e);
console.log("CRAWLER SERVICE: Could not bulkCreate marketalers reason: ", e);
process.exit();
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
console.log("CRAWLER SERVICE: Error crawling. Trying next crawler! ", e);
process.exit();
}
})
};