Files
old-web/app/services/crawlerService.js

46 lines
1.2 KiB
JavaScript
Raw Normal View History

2019-06-13 15:49:31 +02:00
// import OlxCrawler from '../helpers/crawlers/olixClawler'
const OlxCrawler = require("../helpers/crawlers/olxClawler");
2019-06-20 14:51:14 +02:00
const db = require("../models/index");
const MarketAlert = require("../models/marketalert");
2019-06-13 15:49:31 +02:00
const crawlers = [
new OlxCrawler(1, 2, 3),
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
2019-06-13 15:49:31 +02:00
];
async function crawlAll() {
2019-06-20 14:51:14 +02:00
console.log(db.MarketAlert);
2019-06-13 15:49:31 +02:00
for (let crawler of crawlers) {
try {
2019-06-20 14:51:14 +02:00
let results = await crawler.crawl();
for (const result of results) {
const newMAlert = Object.assign({}, MarketAlert)
console.log(newMAlert);
db.MarketAlert.bulkCreate( [{
}])
2019-06-13 15:49:31 +02:00
}
2019-06-20 14:51:14 +02:00
// for (let saver of savers) {
// try {
// await saver.connect();
// await saver.save(results);
// } catch (e) {
// console.log("Error saving. Trying next saver! ", e);
// }
// }
2019-06-13 15:49:31 +02:00
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
}
2019-06-20 14:51:14 +02:00
2019-06-13 15:49:31 +02:00
for (let saver of savers) {
saver.close();
}
}
crawlAll();