senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Preparing to save results to db

Browse Source
This commit is contained in:
Nedim Uka
2019-06-20 14:51:14 +02:00
parent c8ee848f0e
commit 1bcc5e8e5d
5 changed files with 40 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/helpers/crawlers/olxClawler.js
View File

@@ -110,8 +110,6 @@ module.exports = class OlxCrawler {
return null; return null;
} }
async indexPage(url, maxResults = 1000) { async indexPage(url, maxResults = 1000) {
try { try {
// console.log('Starting to index page: ' + pageNr); // console.log('Starting to index page: ' + pageNr);
@@ -138,7 +136,7 @@ module.exports = class OlxCrawler {
if (singleData) { if (singleData) {
results.push(singleData); results.push(singleData);
} }
await this.sleep(500); // await this.sleep(500);
} }
return results; return results;
@@ -177,7 +175,7 @@ module.exports = class OlxCrawler {
let result = await this.indexPage(url, maxResults); let result = await this.indexPage(url, maxResults);
// Object.assign(results, result) // Object.assign(results, result)
results.push(result); results.push(result);
await this.sleep(5000); // await this.sleep(5000);
} }
return results; return results;
} }

10
app/helpers/db/dbHelper.js Normal file
View File

@@ -0,0 +1,10 @@
// const db = require('../../models/index');
// const bulkInsert = async (reuslts) => {
// db.MarketAlert.bulkCreate({
// })
// }

2
app/helpers/url.js
View File

@@ -7,7 +7,7 @@ const currentRERequest = async (req) => {
const request = await db.RealEstateRequest.findOne({ where: {uniqueId} }); const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
return request; return request;
}; };
// TODO Fetch only subscribed realestate requests
const allRERequest = async () => { const allRERequest = async () => {
return await db.RealEstateRequest.findAll(); return await db.RealEstateRequest.findAll();
} }

6
app/models/marketalert.js
View File

@@ -5,6 +5,12 @@ module.exports = (sequelize, DataTypes) => {
realestateOrigin: DataTypes.STRING, realestateOrigin: DataTypes.STRING,
originId: DataTypes.STRING, originId: DataTypes.STRING,
lastDate: DataTypes.STRING, lastDate: DataTypes.STRING,
size : DataTypes.INTEGER,
gardenSize : DataTypes.INTEGER,
price : DataTypes.INTEGER,
municipailty : DataTypes.STRING,
region : DataTypes.STRING,
email: { email: {
type: DataTypes.STRING, type: DataTypes.STRING,
allowNul: false allowNul: false

29
app/services/crawlerService.js
View File

@@ -1,6 +1,8 @@
// import OlxCrawler from '../helpers/crawlers/olixClawler' // import OlxCrawler from '../helpers/crawlers/olixClawler'
const OlxCrawler = require("../helpers/crawlers/olxClawler"); const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const MarketAlert = require("../models/marketalert");
const crawlers = [ const crawlers = [
new OlxCrawler(1, 2, 3), new OlxCrawler(1, 2, 3),
@@ -8,22 +10,33 @@ const crawlers = [
]; ];
async function crawlAll() { async function crawlAll() {
console.log(db.MarketAlert);
for (let crawler of crawlers) { for (let crawler of crawlers) {
try { try {
let results = await crawler.crawl() let results = await crawler.crawl();
for (let saver of savers) {
try { for (const result of results) {
await saver.connect();
await saver.save(results); const newMAlert = Object.assign({}, MarketAlert)
} catch (e) { console.log(newMAlert);
console.log("Error saving. Trying next saver! ", e); db.MarketAlert.bulkCreate( [{
}
}])
} }
// for (let saver of savers) {
// try {
// await saver.connect();
// await saver.save(results);
// } catch (e) {
// console.log("Error saving. Trying next saver! ", e);
// }
// }
} catch (e) { } catch (e) {
console.log("Error crawling. Trying next crawler! ", e); console.log("Error crawling. Trying next crawler! ", e);
} }
} }
for (let saver of savers) { for (let saver of savers) {
saver.close(); saver.close();