senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
Nedim Uka
2019-06-20 21:27:51 +02:00
parent 1bcc5e8e5d
commit 3c59292f23
1 changed files with 54 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
app/services/crawlerService.js
View File

@@ -5,42 +5,65 @@ const db = require("../models/index");
const MarketAlert = require("../models/marketalert"); const MarketAlert = require("../models/marketalert");
const crawlers = [ const crawlers = [
new OlxCrawler(1, 2, 3), new OlxCrawler(1, 2, 3),
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS), // new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
]; ];
async function crawlAll() { async function crawlAll() {
console.log(db.MarketAlert); const properties = db.MarketAlert.rawAttributes;
console.log(properties);
for (let crawler of crawlers) { for (let crawler of crawlers) {
try { try {
let results = await crawler.crawl(); let results = await crawler.crawl();
const marketAlerts = [];
for (const result of results) { for (const result of results) {
console.log("This is result", result);
console.log("This is result", result.size);
const newMAlert = Object.assign({}, MarketAlert) // category: category,
console.log(newMAlert); // url,
db.MarketAlert.bulkCreate( [{ // title,
// price: isNaN(parsedPrice) ? price : parsedPrice,
// size: parseFloat(size),
// rooms: parsedRooms,
// floor: parseInt(floor),
// address,
// location,
// // adType: AD_TYPE_SALE,
// time,
// shortDescription: descriptions.first().text(),
// longDescription: descriptions.last().text(),
// lat,
// lng,
// loc: [parseFloat(lat), parseFloat(lng)],
}]) marketAlerts.push({
} url: result.url,
// for (let saver of savers) { realestateOrigin: "OLX",
// try { originId: "1",
// await saver.connect(); size: "" + result.size,
// await saver.save(results); price: result.price,
// } catch (e) { email: "em"
// console.log("Error saving. Trying next saver! ", e); // lastDate: DataTypes.STRING,
// } // municipailty: DataTypes.STRING,
// } // region: DataTypes.STRING,
} catch (e) { // gardenSize: DataTypes.INTEGER,
console.log("Error crawling. Trying next crawler! ", e);
})
} }
}
try {
await db.MarketAlert.bulkCreate(marketAlerts);
for (let saver of savers) { } catch (e) {
saver.close(); console.log("Could not bulkCreate marketalers reason: ", e);
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
} }
} }
}
crawlAll();
crawlAll();