senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'crawler-service' into 'master'

Browse Source 
Crawler service

See merge request saburly/marketalarm/web!17
This commit was merged in pull request #17.
This commit is contained in:
Bilal Catic
2019-06-24 14:09:47 +00:00
parent c15f45e8f4 1aa91fb4e2
commit 5ffdaef1bf
12 changed files with 952 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1042
app/helpers/codes.js
View File

File diff suppressed because it is too large Load Diff

234
app/helpers/crawlers/olxClawler.js Normal file
View File

@@ -0,0 +1,234 @@
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
const { getRealEstateTypeEnum } = require('../enums');
const { getRegion, getMunicipality } = require('../codes')
const Promise = require("bluebird");
module.exports = class OlxCrawler {
//TODO figure best way to handle paging
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
this.fromPage = fromPage;
this.toPage = toPage;
this.maxResults = maxResults;
}
async indexSingle(url, email) {
try {
const res = await fetch(url);
const body = await res.text();
const $ = cheerio.load(body);
//TODO figure out what to do with username
const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
// if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
// return null;
// }
//TODO remove properties that are not needed, and add some if they are missing
const title = $('#naslovartikla').text();
const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
const price = $('#pc > p:nth-child(2)').text();
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
const time = $('time').attr('datetime');
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
const descriptions = $('.artikal_detaljniopis_tekst');
// const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
const imgRe = /href":("[^"]*")/g;
const matches = latLngRe.exec(body);
let lng = '',
lat = '';
const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
const parsePrice = (price) => parseFloat(price.replace(".", ""))
// TODO we dont save images ??
// const images = [];
// const imgMatches = body.match(imgRe);
// for (let i = 0; imgMatches && i < imgMatches.length; i++) {
// let img = imgMatches[i].replace("href\":", "")
// img = img.replace("\"", "");
// img = img.replace("\"", "");
// images.push(img);
// }
// const uploadPromises = images.map(img => {
// const imgFixed = eval(`'${img}'`);
// return cloudinary.uploader.upload(eval(`'${img}'`));
// });
// const uploadResults = await Promise.all(uploadPromises);
// const cloudinaryImages = uploadResults.map(ur => ur.url);
if (matches && matches.length >= 3) {
lat = matches[1];
lng = matches[2];
}
const parsedPrice = parsePrice(price);
const locationArray = location.split(",");
const region = locationArray[0];
const municipality = locationArray[1];
const data = {
realEstateType: this.getCategoryId(realEstateType),
email : email,
olxId: olxId,
// category: category,
url,
title,
price: isNaN(parsedPrice) ? 0 : parsedPrice,
size: parseFloat(size),
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
address,
region,
municipality,
// adType: AD_TYPE_SALE,
time,
shortDescription: descriptions.first().text(),
longDescription: descriptions.last().text(),
lat,
lng,
loc: [parseFloat(lat), parseFloat(lng)],
// images: cloudinaryImages
};
return data;
} catch (e) {
console.error('Exception caught: ' + e.message);
}
return null;
}
async indexPage(olxUrl, maxResults = 1000) {
try {
//TODO fix paging
// console.log('Starting to index page: ' + pageNr);
// const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
const res = await fetch(olxUrl.url);
const body = await res.text();
const $ = cheerio.load(body);
const hrefs = [];
const results = [];
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
const href = $(elem).find('a').first().attr('href');
hrefs.push(href);
});
let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
for (let i = 0; i < hrefs.length; i++) {
console.log(`indexing: ${hrefs[i]}`);
const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
if (singleData) {
results.push(singleData);
}
}
return results;
} catch (e) {
console.error('Exception caught:' + e);
}
}
getCategoryId (category) {
switch(category) {
case 'Stanovi':
return 'stan';
case 'Vikendice':
return 'vikendica'
case 'Kuće':
return 'kuca';
default:
return '';
}
}
async indexPages(urls, start, end, maxResults = 1000) {
//TODO fix paging
// let results = {};
// for (let i = start; i <= end; i++) {
// let result = await this.indexPage(i, maxResults);
// Object.assign(results, result)
// }
// return results;
let results = [];
for (let url of urls) {
let result = await this.indexPage(url, maxResults);
results.push(result);
}
return results;
}
async crawl() {
const filteredResults = [];
const realestateRequests = await allRERequest()
const urls = this.createRequestUrls(realestateRequests);
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
for (const result of results) {
for (const finalResult of result) {
if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]);
if (pointInsideBoundingBox[0].length !== 0) {
filteredResults.push(finalResult);
}
}
}
}
console.log(filteredResults);
return filteredResults;
}
createRequestUrls(realestateRequests) {
const urls = []
for (const request of realestateRequests) {
const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
const region = "kanton=" + getRegion(request.region).olxid;
const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
const sizeMin = "kvadrata_min=" + request.sizeMin;
const sizeMax = "kvadrata_max=" + request.sizeMax;
const priceMin = "od=" + request.priceMin;
const priceMax = "do=" + request.priceMax;
const olxUrl = {
url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
email: request.email
}
console.log(olxUrl.url);
urls.push(olxUrl);
}
return urls;
}
};

15
app/helpers/db/dbHelper.js Normal file
View File

@@ -0,0 +1,15 @@
const db = require('../../models/index');
// TODO Fetch only subscribed realestate requests
const allRERequest = async () => {
return await db.RealEstateRequest.findAll();
}
const findPointInsideBoundingBox = async (latLng) => {
return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
}
module.exports = {
allRERequest,
findPointInsideBoundingBox
};

6
app/helpers/enums.js
View File

@@ -1,7 +1,7 @@
const realEstateTypes = [
{ title: "Kuća", id: "kuca", hasGardenSize: true },
{ title: "Stan", id: "stan", hasGardenSize: false },
{ title: "Vikendica", id: "vikendica", hasGardenSize: true }
{ title: "Kuća", id: "kuca", hasGardenSize: true, olxCategory: 24 },
{ title: "Stan", id: "stan", hasGardenSize: false, olxCategory: 23},
{ title: "Vikendica", id: "vikendica", hasGardenSize: true, olxCategory: 26 }
];
const sizes = [

12
app/helpers/url.js
View File

@@ -7,7 +7,17 @@ const currentRERequest = async (req) => {
const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
return request;
};
// TODO Fetch only subscribed realestate requests
const allRERequest = async () => {
return await db.RealEstateRequest.findAll();
}
const findPointInsideBoundingBox = async (latLng) => {
return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
}
module.exports = {
currentRERequest
currentRERequest,
allRERequest,
findPointInsideBoundingBox
};

37
app/migrations/20190618103020-expand-maketalert.js Normal file
View File

@@ -0,0 +1,37 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction((t) => {
return Promise.all([
queryInterface.addColumn('MarketAlerts', 'size', {
type: Sequelize.INTEGER,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'gardenSize', {
type: Sequelize.INTEGER,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'price', {
type: Sequelize.INTEGER,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'municipality', {
type: Sequelize.STRING,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'region', {
type: Sequelize.STRING,
}, { transaction: t })
])
})
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction((t) => {
return Promise.all([
queryInterface.removeColumn('MarketAlerts', 'size', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'gardenSize', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'price', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'municipality', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'region', { transaction: t })
])
})
}
};

33
app/migrations/20190618124522-marketalerts-additional-info.js Normal file
View File

@@ -0,0 +1,33 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction((t) => {
return Promise.all([
queryInterface.removeColumn('MarketAlerts', 'olxUrl', { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'url', {
type: Sequelize.STRING,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'realestateOrigin', {
type: Sequelize.STRING,
}, { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'originId', {
type: Sequelize.STRING,
}, { transaction: t })
])
})
},
down: (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction((t) => {
return Promise.all([
queryInterface.removeColumn('MarketAlerts', 'url', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'realestateOrigin', { transaction: t }),
queryInterface.removeColumn('MarketAlerts', 'originId', { transaction: t }),
queryInterface.addColumn('MarketAlerts', 'olxUrl', {
type: Sequelize.STRING
}, { transaction: t })
])
})
}
};

20
app/migrations/20190621162321-add-category-to-marketalert.js Normal file
View File

@@ -0,0 +1,20 @@
'use strict';
module.exports = {
up: (queryInterface, Sequelize) => {
return queryInterface.addColumn(
'MarketAlerts',
'realEstateType',
{
type: Sequelize.STRING
}
);
},
down: (queryInterface, Sequelize) => {
return queryInterface.removeColumn(
'MarketAlerts',
'realEstateType'
);
}
};

11
app/models/marketalert.js
View File

@@ -1,8 +1,17 @@
'use strict';
module.exports = (sequelize, DataTypes) => {
const MarketAlert = sequelize.define('MarketAlert', {
olxUrl: DataTypes.STRING,
url: DataTypes.STRING,
realestateOrigin: DataTypes.STRING,
originId: DataTypes.STRING,
lastDate: DataTypes.STRING,
size : DataTypes.INTEGER,
gardenSize : DataTypes.INTEGER,
price : DataTypes.INTEGER,
municipality : DataTypes.STRING,
region : DataTypes.STRING,
realEstateType : DataTypes.STRING,
email: {
type: DataTypes.STRING,
allowNul: false

55
app/services/crawlerService.js Normal file
View File

@@ -0,0 +1,55 @@
const Promise = require("bluebird");
const OlxCrawler = require("../helpers/crawlers/olxClawler");
const db = require("../models/index");
const olxCrawler = new OlxCrawler(1, 2, 3);
const crawlers = [
olxCrawler,
];
async function crawlAll() {
Promise.map(crawlers, function (crawler) {
return crawler.crawl();
}).then(async (results) => {
try {
const marketAlertsFromDb = await db.MarketAlert.findAll();
const marketAlerts = [];
const mergedResults = [].concat.apply([], results);
for (const result of mergedResults) {
marketAlerts.push({
url: result.url,
realestateOrigin: "OLX",
originId: 1,
size: result.size,
price: result.price,
email: result.email,
// lastDate: DataTypes.STRING,
municipality: result.municipality,
region: result.region,
gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
realEstateType: result.realEstateType
})
}
try {
console.log(marketAlerts);
const filteredMarketAlerts = marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
await db.MarketAlert.bulkCreate(filteredMarketAlerts);
process.exit()
} catch (e) {
console.log("Could not bulkCreate marketalers reason: ", e);
}
} catch (e) {
console.log("Error crawling. Trying next crawler! ", e);
}
})
};
crawlAll();

6
package-lock.json generated
View File

@@ -327,9 +327,9 @@
"dev": true
},
"bluebird": {
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
"integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw=="
"version": "3.5.5",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz",
"integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w=="
},
"body-parser": {
"version": "1.18.3",

2
package.json
View File

@@ -7,6 +7,7 @@
"test": "echo \"Error: no test specified\" && exit 1",
"start": "node ./index.js",
"start-mon": "nodemon ./index.js",
"scheduler": "node ./app/services/crawlerService.js",
"migrate": "cd app && npx sequelize db:migrate",
"setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
"docker-start": "docker start pg_marketalerts",
@@ -25,6 +26,7 @@
"2checkout-node": "0.0.1",
"@sendgrid/mail": "^6.3.1",
"aws-sdk": "^2.422.0",
"bluebird": "^3.5.5",
"cheerio": "^1.0.0-rc.2",
"compression": "^1.7.4",
"dotenv": "^7.0.0",