192 lines
4.7 KiB
JavaScript
192 lines
4.7 KiB
JavaScript
|
|
// import OlxCrawler from '../helpers/crawlers/olixClawler'
|
|
var Promise = require("bluebird");
|
|
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
|
const db = require("../models/index");
|
|
const MarketAlert = require("../models/marketalert");
|
|
|
|
const olxCrawler = new OlxCrawler(1, 2, 3);
|
|
|
|
const crawlers = [
|
|
olxCrawler,
|
|
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
|
|
];
|
|
|
|
async function crawlAll() {
|
|
|
|
Promise.map(crawlers, function (crawler) {
|
|
|
|
return crawler.crawl();
|
|
|
|
}).then(async (results) => {
|
|
// let results = await crawler.crawl();
|
|
try {
|
|
const marketAlerts = [];
|
|
const mergedResults = [].concat.apply([], results);
|
|
|
|
for (const result of mergedResults) {
|
|
|
|
|
|
// category: category,
|
|
// url,
|
|
// title,
|
|
// price: isNaN(parsedPrice) ? price : parsedPrice,
|
|
// size: parseFloat(size),
|
|
// rooms: parsedRooms,
|
|
// floor: parseInt(floor),
|
|
// address,
|
|
// location,
|
|
// // adType: AD_TYPE_SALE,
|
|
// time,
|
|
// shortDescription: descriptions.first().text(),
|
|
// longDescription: descriptions.last().text(),
|
|
// lat,
|
|
// lng,
|
|
// loc: [parseFloat(lat), parseFloat(lng)],
|
|
|
|
|
|
marketAlerts.push({
|
|
url: result.url,
|
|
realestateOrigin: "OLX",
|
|
originId: result.olxId,
|
|
size: result.size,
|
|
price: result.price,
|
|
email: result.email,
|
|
// lastDate: DataTypes.STRING,
|
|
municipality: result.municipality,
|
|
region:result.region,
|
|
// gardenSize: DataTypes.INTEGER,
|
|
|
|
|
|
})
|
|
}
|
|
|
|
try {
|
|
await db.MarketAlert.bulkCreate(marketAlerts);
|
|
} catch (e) {
|
|
console.log("Could not bulkCreate marketalers reason: ", e);
|
|
}
|
|
} catch (e) {
|
|
console.log("Error crawling. Trying next crawler! ", e);
|
|
}
|
|
}
|
|
|
|
|
|
)
|
|
};
|
|
|
|
|
|
|
|
// Promise.all(
|
|
|
|
|
|
// ).then((results) => {
|
|
// console.log(results);
|
|
// console.log(results.length);
|
|
// console.log("Executing save results");
|
|
// Promise.all([extractAndSaveResults(results)]).then(() => {
|
|
// console.log("Executed save results");
|
|
// });
|
|
// });
|
|
|
|
|
|
// for (let crawler of crawlers) {
|
|
// try {
|
|
// let results = await crawler.crawl();
|
|
// const marketAlerts = [];
|
|
|
|
// for (const result of results) {
|
|
// console.log("This is result", result);
|
|
// console.log("This is result", result.size);
|
|
|
|
// // category: category,
|
|
// // url,
|
|
// // title,
|
|
// // price: isNaN(parsedPrice) ? price : parsedPrice,
|
|
// // size: parseFloat(size),
|
|
// // rooms: parsedRooms,
|
|
// // floor: parseInt(floor),
|
|
// // address,
|
|
// // location,
|
|
// // // adType: AD_TYPE_SALE,
|
|
// // time,
|
|
// // shortDescription: descriptions.first().text(),
|
|
// // longDescription: descriptions.last().text(),
|
|
// // lat,
|
|
// // lng,
|
|
// // loc: [parseFloat(lat), parseFloat(lng)],
|
|
|
|
|
|
// marketAlerts.push({
|
|
// url: result.url,
|
|
// realestateOrigin: "OLX",
|
|
// originId: result.olxId,
|
|
// size: result.size,
|
|
// price: result.price,
|
|
// email: "em"
|
|
// // lastDate: DataTypes.STRING,
|
|
// // municipailty: DataTypes.STRING,
|
|
// // region: DataTypes.STRING,
|
|
// // gardenSize: DataTypes.INTEGER,
|
|
|
|
|
|
// })
|
|
// }
|
|
|
|
// try {
|
|
// await db.MarketAlert.bulkCreate(marketAlerts);
|
|
// } catch (e) {
|
|
// console.log("Could not bulkCreate marketalers reason: ", e);
|
|
// }
|
|
// } catch (e) {
|
|
// console.log("Error crawling. Trying next crawler! ", e);
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
async function extractAndSaveResults(results) {
|
|
const marketAlerts = []
|
|
|
|
for (const result of results) {
|
|
// console.log("This is result", result);
|
|
// console.log("This is result", result.size);
|
|
|
|
// category: category,
|
|
// url,
|
|
// title,
|
|
// price: isNaN(parsedPrice) ? price : parsedPrice,
|
|
// size: parseFloat(size),
|
|
// rooms: parsedRooms,
|
|
// floor: parseInt(floor),
|
|
// address,
|
|
// location,
|
|
// // adType: AD_TYPE_SALE,
|
|
// time,
|
|
// shortDescription: descriptions.first().text(),
|
|
// longDescription: descriptions.last().text(),
|
|
// lat,
|
|
// lng,
|
|
// loc: [parseFloat(lat), parseFloat(lng)],
|
|
|
|
|
|
marketAlerts.push({
|
|
url: result.url,
|
|
realestateOrigin: "OLX",
|
|
originId: result.olxId,
|
|
size: result.size,
|
|
price: result.price,
|
|
email: "em"
|
|
// lastDate: DataTypes.STRING,
|
|
// municipailty: DataTypes.STRING,
|
|
// region: DataTypes.STRING,
|
|
// gardenSize: DataTypes.INTEGER,
|
|
|
|
|
|
})
|
|
}
|
|
|
|
return marketAlerts;
|
|
|
|
}
|
|
|
|
crawlAll(); |