69 lines
1.9 KiB
JavaScript
69 lines
1.9 KiB
JavaScript
|
|
// import OlxCrawler from '../helpers/crawlers/olixClawler'
|
|
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
|
const db = require("../models/index");
|
|
const MarketAlert = require("../models/marketalert");
|
|
|
|
const crawlers = [
|
|
new OlxCrawler(1, 2, 3),
|
|
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
|
|
];
|
|
|
|
async function crawlAll() {
|
|
const properties = db.MarketAlert.rawAttributes;
|
|
console.log(properties);
|
|
|
|
for (let crawler of crawlers) {
|
|
try {
|
|
let results = await crawler.crawl();
|
|
const marketAlerts = [];
|
|
|
|
for (const result of results) {
|
|
console.log("This is result", result);
|
|
console.log("This is result", result.size);
|
|
|
|
// category: category,
|
|
// url,
|
|
// title,
|
|
// price: isNaN(parsedPrice) ? price : parsedPrice,
|
|
// size: parseFloat(size),
|
|
// rooms: parsedRooms,
|
|
// floor: parseInt(floor),
|
|
// address,
|
|
// location,
|
|
// // adType: AD_TYPE_SALE,
|
|
// time,
|
|
// shortDescription: descriptions.first().text(),
|
|
// longDescription: descriptions.last().text(),
|
|
// lat,
|
|
// lng,
|
|
// loc: [parseFloat(lat), parseFloat(lng)],
|
|
|
|
marketAlerts.push({
|
|
url: result.url,
|
|
realestateOrigin: "OLX",
|
|
originId: "1",
|
|
size: "" + result.size,
|
|
price: result.price,
|
|
email: "em"
|
|
// lastDate: DataTypes.STRING,
|
|
// municipailty: DataTypes.STRING,
|
|
// region: DataTypes.STRING,
|
|
// gardenSize: DataTypes.INTEGER,
|
|
|
|
|
|
})
|
|
}
|
|
|
|
try {
|
|
await db.MarketAlert.bulkCreate(marketAlerts);
|
|
} catch (e) {
|
|
console.log("Could not bulkCreate marketalers reason: ", e);
|
|
}
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
crawlAll(); |