Olix crawling, filter crawling result by lat, lng
This commit is contained in:
Nedim Uka
1042
app/helpers/codes.js
1042
app/helpers/codes.js
File diff suppressed because it is too large
Load Diff
@@ -1,163 +0,0 @@
|
||||
const fetch = require('node-fetch');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
export default class OlxCrawler {
|
||||
|
||||
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
||||
this.fromPage = fromPage;
|
||||
this.toPage = toPage;
|
||||
this.maxResults = maxResults;
|
||||
}
|
||||
|
||||
async indexSingle(url) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
|
||||
|
||||
if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const title = $('#naslovartikla').text();
|
||||
const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
|
||||
|
||||
const price = $('#pc > p:nth-child(2)').text();
|
||||
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
|
||||
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
|
||||
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
|
||||
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
|
||||
|
||||
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
|
||||
const time = $('time').attr('datetime');
|
||||
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
|
||||
|
||||
const descriptions = $('.artikal_detaljniopis_tekst');
|
||||
const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
|
||||
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const imgRe = /href":("[^"]*")/g;
|
||||
const matches = latLngRe.exec(body);
|
||||
let lng = '',
|
||||
lat = '';
|
||||
|
||||
const images = [];
|
||||
const imgMatches = body.match(imgRe);
|
||||
|
||||
const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
|
||||
const parsePrice = (price) => parseFloat(price.replace(".", ""))
|
||||
|
||||
|
||||
for (let i = 0; imgMatches && i < imgMatches.length; i++) {
|
||||
let img = imgMatches[i].replace("href\":", "")
|
||||
img = img.replace("\"", "");
|
||||
img = img.replace("\"", "");
|
||||
images.push(img);
|
||||
}
|
||||
|
||||
const uploadPromises = images.map(img => {
|
||||
const imgFixed = eval(`'${img}'`);
|
||||
return cloudinary.uploader.upload(eval(`'${img}'`));
|
||||
});
|
||||
|
||||
const uploadResults = await Promise.all(uploadPromises);
|
||||
const cloudinaryImages = uploadResults.map(ur => ur.url);
|
||||
|
||||
if (matches && matches.length >= 3) {
|
||||
lat = matches[1];
|
||||
lng = matches[2];
|
||||
}
|
||||
|
||||
const parsedPrice = parsePrice(price);
|
||||
let parsedRooms;
|
||||
|
||||
if (rooms === 'Garsonjera') {
|
||||
parsedRooms = 0;
|
||||
} else {
|
||||
parsedRooms = parseRooms(rooms);
|
||||
}
|
||||
|
||||
const data = {
|
||||
category: this.getCategoryId(category),
|
||||
url,
|
||||
title,
|
||||
price: isNaN(parsedPrice) ? price : parsedPrice,
|
||||
size: parseFloat(size),
|
||||
rooms: parsedRooms,
|
||||
floor: parseInt(floor),
|
||||
address,
|
||||
location,
|
||||
adType: AD_TYPE_SALE,
|
||||
time,
|
||||
shortDescription: descriptions.first().text(),
|
||||
longDescription: descriptions.last().text(),
|
||||
lat,
|
||||
lng,
|
||||
loc: [parseFloat(lat), parseFloat(lng)],
|
||||
images: cloudinaryImages
|
||||
};
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
console.error('Exception caught: ' + e.message);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async indexPage(url, pageNr, maxResults = 1000) {
|
||||
try {
|
||||
console.log('Starting to index page: ' + pageNr);
|
||||
const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
|
||||
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
const hrefs = [];
|
||||
const results = {};
|
||||
|
||||
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
|
||||
const href = $(elem).find('a').first().attr('href');
|
||||
hrefs.push(href);
|
||||
});
|
||||
|
||||
let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
|
||||
|
||||
for (let i = 0; i < hrefs.length; i++) {
|
||||
console.log(`indexing: ${hrefs[i]}`);
|
||||
|
||||
const singleData = await this.indexSingle(hrefs[i]);
|
||||
|
||||
if (singleData) {
|
||||
results[hrefs[i]] = singleData;
|
||||
}
|
||||
await this.sleep(500);
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (e) {
|
||||
console.error('Exception caught:' + e);
|
||||
}
|
||||
}
|
||||
|
||||
async sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async indexPages(url, start, end, maxResults = 1000) {
|
||||
let results = {};
|
||||
for (let i = start; i <= end; i++) {
|
||||
let result = await this.indexPage(i, maxResults);
|
||||
Object.assign(results, result)
|
||||
await this.sleep(5000);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
// TODO create URLS from db
|
||||
let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
|
||||
return results;
|
||||
}
|
||||
}
|
||||
230
app/helpers/crawlers/olxClawler.js
Normal file
230
app/helpers/crawlers/olxClawler.js
Normal file
@@ -0,0 +1,230 @@
|
||||
const fetch = require('node-fetch');
|
||||
const cheerio = require('cheerio');
|
||||
const { allRERequest, findPointInsideBoundingBox } = require('../url');
|
||||
const { getRealEstateTypeEnum } = require('../enums');
|
||||
const { getRegion, getMunicipality } = require('../codes')
|
||||
|
||||
module.exports = class OlxCrawler {
|
||||
|
||||
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
||||
this.fromPage = fromPage;
|
||||
this.toPage = toPage;
|
||||
this.maxResults = maxResults;
|
||||
}
|
||||
|
||||
async indexSingle(url) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
|
||||
|
||||
// if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
|
||||
// return null;
|
||||
// }
|
||||
|
||||
const title = $('#naslovartikla').text();
|
||||
const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
|
||||
|
||||
const price = $('#pc > p:nth-child(2)').text();
|
||||
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
|
||||
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
|
||||
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
|
||||
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
|
||||
|
||||
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
|
||||
const time = $('time').attr('datetime');
|
||||
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
|
||||
|
||||
const descriptions = $('.artikal_detaljniopis_tekst');
|
||||
const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
|
||||
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
||||
const imgRe = /href":("[^"]*")/g;
|
||||
const matches = latLngRe.exec(body);
|
||||
let lng = '',
|
||||
lat = '';
|
||||
|
||||
const images = [];
|
||||
const imgMatches = body.match(imgRe);
|
||||
|
||||
const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
|
||||
const parsePrice = (price) => parseFloat(price.replace(".", ""))
|
||||
|
||||
|
||||
for (let i = 0; imgMatches && i < imgMatches.length; i++) {
|
||||
let img = imgMatches[i].replace("href\":", "")
|
||||
img = img.replace("\"", "");
|
||||
img = img.replace("\"", "");
|
||||
images.push(img);
|
||||
}
|
||||
|
||||
// const uploadPromises = images.map(img => {
|
||||
// const imgFixed = eval(`'${img}'`);
|
||||
// return cloudinary.uploader.upload(eval(`'${img}'`));
|
||||
// });
|
||||
|
||||
// const uploadResults = await Promise.all(uploadPromises);
|
||||
// const cloudinaryImages = uploadResults.map(ur => ur.url);
|
||||
|
||||
if (matches && matches.length >= 3) {
|
||||
lat = matches[1];
|
||||
lng = matches[2];
|
||||
}
|
||||
|
||||
const parsedPrice = parsePrice(price);
|
||||
let parsedRooms;
|
||||
|
||||
if (rooms === 'Garsonjera') {
|
||||
parsedRooms = 0;
|
||||
} else {
|
||||
parsedRooms = parseRooms(rooms);
|
||||
}
|
||||
|
||||
const data = {
|
||||
// category: this.getCategoryId(category),
|
||||
category: category,
|
||||
url,
|
||||
title,
|
||||
price: isNaN(parsedPrice) ? price : parsedPrice,
|
||||
size: parseFloat(size),
|
||||
rooms: parsedRooms,
|
||||
floor: parseInt(floor),
|
||||
address,
|
||||
location,
|
||||
// adType: AD_TYPE_SALE,
|
||||
time,
|
||||
shortDescription: descriptions.first().text(),
|
||||
longDescription: descriptions.last().text(),
|
||||
lat,
|
||||
lng,
|
||||
loc: [parseFloat(lat), parseFloat(lng)],
|
||||
// images: cloudinaryImages
|
||||
};
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
console.error('Exception caught: ' + e.message);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
async indexPage(url, maxResults = 1000) {
|
||||
try {
|
||||
// console.log('Starting to index page: ' + pageNr);
|
||||
// const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
|
||||
|
||||
const res = await fetch(url);
|
||||
const body = await res.text();
|
||||
const $ = cheerio.load(body);
|
||||
const hrefs = [];
|
||||
const results = [];
|
||||
|
||||
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
|
||||
const href = $(elem).find('a').first().attr('href');
|
||||
hrefs.push(href);
|
||||
});
|
||||
|
||||
let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
|
||||
|
||||
for (let i = 0; i < hrefs.length; i++) {
|
||||
console.log(`indexing: ${hrefs[i]}`);
|
||||
|
||||
const singleData = await this.indexSingle(hrefs[i]);
|
||||
|
||||
if (singleData) {
|
||||
results.push(singleData);
|
||||
}
|
||||
await this.sleep(500);
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (e) {
|
||||
console.error('Exception caught:' + e);
|
||||
}
|
||||
}
|
||||
|
||||
// getCategoryId (category) {
|
||||
// if (category === 'Stanovi') {
|
||||
// return CATEGORY_FLAT;
|
||||
// } else if (category === 'Zemljišta') {
|
||||
// return CATEGORY_LAND;
|
||||
// } else if (category === 'Kuće') {
|
||||
// return CATEGORY_HOUSE;
|
||||
// } else if (category === 'Poslovni prostori') {
|
||||
// return CATEGORY_OFFICE;
|
||||
// }
|
||||
// }
|
||||
|
||||
async sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async indexPages(urls, start, end, maxResults = 1000) {
|
||||
// let results = {};
|
||||
// for (let i = start; i <= end; i++) {
|
||||
// let result = await this.indexPage(i, maxResults);
|
||||
// Object.assign(results, result)
|
||||
// await this.sleep(5000);
|
||||
// }
|
||||
// return results;
|
||||
|
||||
let results = [];
|
||||
for (let url of urls) {
|
||||
let result = await this.indexPage(url, maxResults);
|
||||
// Object.assign(results, result)
|
||||
results.push(result);
|
||||
await this.sleep(5000);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async crawl() {
|
||||
// TODO create URLS from db
|
||||
const filteredResults = [];
|
||||
const realestateRequests = await allRERequest()
|
||||
const urls = this.createRequestUrls(realestateRequests);
|
||||
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
|
||||
|
||||
for (const result of results) {
|
||||
console.log(result);
|
||||
for (const re1 of result) {
|
||||
if (re1.lat !== undefined) {
|
||||
console.log(re1.lat);
|
||||
const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
|
||||
console.log(pointInsideBoundingBox);
|
||||
}
|
||||
}
|
||||
}
|
||||
// console.log(results);
|
||||
|
||||
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
createRequestUrls(realestateRequests) {
|
||||
const urls = []
|
||||
|
||||
for (const request of realestateRequests) {
|
||||
const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
|
||||
const region = "kanton=" + getRegion(request.region).olxid;
|
||||
const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
|
||||
const sizeMin = "kvadrata_min=" + request.sizeMin;
|
||||
const sizeMax = "kvadrata_max=" + request.sizeMax;
|
||||
const priceMin = "od=" + request.priceMin;
|
||||
const priceMax = "do=" + request.priceMax;
|
||||
|
||||
|
||||
|
||||
const olxUrl = "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax
|
||||
console.log(olxUrl);
|
||||
urls.push(olxUrl);
|
||||
}
|
||||
|
||||
return urls;
|
||||
}
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
const realEstateTypes = [
|
||||
{ title: "Kuća", id: "kuca", hasGardenSize: true, olixCategory: 23 },
|
||||
{ title: "Stan", id: "stan", hasGardenSize: false, olixCategory: 24},
|
||||
{ title: "Vikendica", id: "vikendica", hasGardenSize: true, olixCategory: 26 }
|
||||
{ title: "Kuća", id: "kuca", hasGardenSize: true, olxCategory: 24 },
|
||||
{ title: "Stan", id: "stan", hasGardenSize: false, olxCategory: 23},
|
||||
{ title: "Vikendica", id: "vikendica", hasGardenSize: true, olxCategory: 26 }
|
||||
];
|
||||
|
||||
const sizes = [
|
||||
|
||||
@@ -8,6 +8,16 @@ const currentRERequest = async (req) => {
|
||||
return request;
|
||||
};
|
||||
|
||||
const allRERequest = async () => {
|
||||
return await db.RealEstateRequest.findAll();
|
||||
}
|
||||
|
||||
const findPointInsideBoundingBox = async (latLng) => {
|
||||
return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
currentRERequest
|
||||
currentRERequest,
|
||||
allRERequest,
|
||||
findPointInsideBoundingBox
|
||||
};
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
|
||||
// import OlxCrawler from '../helpers/crawlers/olixClawler'
|
||||
const OlxCrawler = require("../helpers/crawlers/olxClawler");
|
||||
|
||||
var http = require('http');
|
||||
const crawlers = [
|
||||
//new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
|
||||
new OlxCrawler(1, 2, 3),
|
||||
// new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
|
||||
];
|
||||
|
||||
async function crawlAll() {
|
||||
|
||||
Reference in New Issue
Block a user