senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Compare crawler results with db, and only save new if necessary

Browse Source
This commit is contained in:
Nedim Uka
2019-06-21 16:48:19 +02:00
parent 80ff9bcb6b
commit 2f474619ca
4 changed files with 86 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
app/services/crawlerService.js
View File

@@ -1,192 +1,54 @@
// import OlxCrawler from '../helpers/crawlers/olixClawler'
var Promise = require("bluebird");
const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const MarketAlert = require("../models/marketalert");
const olxCrawler = new OlxCrawler(1, 2, 3);
const crawlers = [
olxCrawler,
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
olxCrawler,
];
async function crawlAll() {
Promise.map(crawlers, function (crawler) {
Promise.map(crawlers, function (crawler) {
return crawler.crawl();
}).then(async (results) => {
return crawler.crawl();
try {
}).then(async (results) => {
// let results = await crawler.crawl();
try {
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
const marketAlertsFromDb = await db.MarketAlert.findAll();
for (const result of mergedResults) {
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
// category: category,
// url,
// title,
// price: isNaN(parsedPrice) ? price : parsedPrice,
// size: parseFloat(size),
// rooms: parsedRooms,
// floor: parseInt(floor),
// address,
// location,
// // adType: AD_TYPE_SALE,
// time,
// shortDescription: descriptions.first().text(),
// longDescription: descriptions.last().text(),
// lat,
// lng,
// loc: [parseFloat(lat), parseFloat(lng)],
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: result.olxId,
size: result.size,
price: result.price,
email: result.email,
// lastDate: DataTypes.STRING,
municipality: result.municipality,
region:result.region,
// gardenSize: DataTypes.INTEGER,
})
}
try {
await db.MarketAlert.bulkCreate(marketAlerts);
} catch (e) {
console.log("Could not bulkCreate marketalers reason: ", e);
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
}
)
for (const result of mergedResults) {
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: result.olxId,
size: result.size,
price: result.price,
email: result.email,
// lastDate: DataTypes.STRING,
municipality: result.municipality,
region: result.region,
gardenSize: result.gardenSize,
realEstateType: result.realEstateType
})
}
try {
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
process.exit()
} catch (e) {
console.log("Could not bulkCreate marketalers reason: ", e);
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
})
};
crawlAll();
// Promise.all(
// ).then((results) => {
// console.log(results);
// console.log(results.length);
// console.log("Executing save results");
// Promise.all([extractAndSaveResults(results)]).then(() => {
// console.log("Executed save results");
// });
// });
// for (let crawler of crawlers) {
// try {
// let results = await crawler.crawl();
// const marketAlerts = [];
// for (const result of results) {
// console.log("This is result", result);
// console.log("This is result", result.size);
// // category: category,
// // url,
// // title,
// // price: isNaN(parsedPrice) ? price : parsedPrice,
// // size: parseFloat(size),
// // rooms: parsedRooms,
// // floor: parseInt(floor),
// // address,
// // location,
// // // adType: AD_TYPE_SALE,
// // time,
// // shortDescription: descriptions.first().text(),
// // longDescription: descriptions.last().text(),
// // lat,
// // lng,
// // loc: [parseFloat(lat), parseFloat(lng)],
// marketAlerts.push({
// url: result.url,
// realestateOrigin: "OLX",
// originId: result.olxId,
// size: result.size,
// price: result.price,
// email: "em"
// // lastDate: DataTypes.STRING,
// // municipailty: DataTypes.STRING,
// // region: DataTypes.STRING,
// // gardenSize: DataTypes.INTEGER,
// })
// }
// try {
// await db.MarketAlert.bulkCreate(marketAlerts);
// } catch (e) {
// console.log("Could not bulkCreate marketalers reason: ", e);
// }
// } catch (e) {
// console.log("Error crawling. Trying next crawler! ", e);
// }
// }
// }
async function extractAndSaveResults(results) {
const marketAlerts = []
for (const result of results) {
// console.log("This is result", result);
// console.log("This is result", result.size);
// category: category,
// url,
// title,
// price: isNaN(parsedPrice) ? price : parsedPrice,
// size: parseFloat(size),
// rooms: parsedRooms,
// floor: parseInt(floor),
// address,
// location,
// // adType: AD_TYPE_SALE,
// time,
// shortDescription: descriptions.first().text(),
// longDescription: descriptions.last().text(),
// lat,
// lng,
// loc: [parseFloat(lat), parseFloat(lng)],
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: result.olxId,
size: result.size,
price: result.price,
email: "em"
// lastDate: DataTypes.STRING,
// municipailty: DataTypes.STRING,
// region: DataTypes.STRING,
// gardenSize: DataTypes.INTEGER,
})
}
return marketAlerts;
}
crawlAll();