562 lines
22 KiB
JavaScript
562 lines
22 KiB
JavaScript
const fetch = require('node-fetch');
|
|
const cheerio = require('cheerio');
|
|
const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
|
|
const { getRealEstateTypeEnum } = require('../enums');
|
|
const { getRegion, getMunicipality } = require('../codes')
|
|
const Promise = require("bluebird");
|
|
|
|
module.exports = class OlxCrawler {
|
|
//TODO figure best way to handle paging
|
|
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
|
this.fromPage = fromPage;
|
|
this.toPage = toPage;
|
|
this.maxResults = maxResults;
|
|
}
|
|
|
|
async indexSingle(url, email, uuid) {
|
|
try {
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
|
|
//TODO figure out what to do with username
|
|
const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
|
|
|
|
// if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
|
|
// return null;
|
|
// }
|
|
|
|
//TODO remove properties that are not needed, and add some if they are missing
|
|
const title = $('#naslovartikla').text().trim();
|
|
const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
|
|
|
|
const price = $('#pc > p:nth-child(2)').text();
|
|
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
|
|
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
|
|
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
|
|
const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
|
|
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
|
|
|
|
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
|
|
const time = $('time').attr('datetime');
|
|
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
|
|
|
|
const descriptions = $('.artikal_detaljniopis_tekst');
|
|
// const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
|
|
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
|
const imgRe = /href":("[^"]*")/g;
|
|
const matches = latLngRe.exec(body);
|
|
let lng = '',
|
|
lat = '';
|
|
|
|
|
|
const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
|
|
const parsePrice = (price) => parseFloat(price.replace(".", ""))
|
|
|
|
|
|
// TODO we dont save images ??
|
|
|
|
// const images = [];
|
|
// const imgMatches = body.match(imgRe);
|
|
|
|
// for (let i = 0; imgMatches && i < imgMatches.length; i++) {
|
|
// let img = imgMatches[i].replace("href\":", "")
|
|
// img = img.replace("\"", "");
|
|
// img = img.replace("\"", "");
|
|
// images.push(img);
|
|
// }
|
|
|
|
// const uploadPromises = images.map(img => {
|
|
// const imgFixed = eval(`'${img}'`);
|
|
// return cloudinary.uploader.upload(eval(`'${img}'`));
|
|
// });
|
|
|
|
// const uploadResults = await Promise.all(uploadPromises);
|
|
// const cloudinaryImages = uploadResults.map(ur => ur.url);
|
|
|
|
if (matches && matches.length >= 3) {
|
|
lat = matches[1];
|
|
lng = matches[2];
|
|
}
|
|
|
|
const parsedPrice = parsePrice(price);
|
|
|
|
console.log(location);
|
|
const locationArray = location.split(",");
|
|
const region = locationArray[0];
|
|
const municipality = locationArray[1];
|
|
|
|
const data = {
|
|
realEstateType: this.getCategoryId(realEstateType),
|
|
email: email,
|
|
uuid: uuid,
|
|
olxId: olxId,
|
|
// category: category,
|
|
url,
|
|
title,
|
|
price: isNaN(parsedPrice) ? 0 : parsedPrice,
|
|
size: parseFloat(size),
|
|
gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
|
|
address,
|
|
region,
|
|
municipality,
|
|
// adType: AD_TYPE_SALE,
|
|
time,
|
|
shortDescription: descriptions.first().text(),
|
|
longDescription: descriptions.last().text(),
|
|
lat,
|
|
lng,
|
|
loc: [parseFloat(lat), parseFloat(lng)],
|
|
// images: cloudinaryImages
|
|
};
|
|
|
|
return data;
|
|
} catch (e) {
|
|
console.error('Exception caught: ' + e.message);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async indexPage(olxUrl, maxResults = 1000) {
|
|
try {
|
|
//TODO fix paging
|
|
// console.log('Starting to index page: ' + pageNr);
|
|
// const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
|
|
|
|
const res = await fetch(olxUrl.url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
const hrefs = [];
|
|
const results = [];
|
|
|
|
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
|
|
const href = $(elem).find('a').first().attr('href');
|
|
hrefs.push(href);
|
|
});
|
|
|
|
let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
|
|
|
|
for (let i = 0; i < hrefs.length; i++) {
|
|
console.log(`indexing: ${hrefs[i]}`);
|
|
|
|
const singleData = await this.indexSingle(hrefs[i], olxUrl.email, olxUrl.uuid);
|
|
|
|
if (singleData) {
|
|
results.push(singleData);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
} catch (e) {
|
|
console.error('Exception caught:' + e);
|
|
}
|
|
}
|
|
|
|
getCategoryId(category) {
|
|
|
|
switch (category) {
|
|
case 'Stanovi':
|
|
return 'stan';
|
|
|
|
case 'Vikendice':
|
|
return 'vikendica'
|
|
|
|
case 'Kuće':
|
|
return 'kuca';
|
|
|
|
default:
|
|
return '';
|
|
}
|
|
}
|
|
|
|
async indexPages(urls, start, end, maxResults = 1000) {
|
|
//TODO fix paging
|
|
// let results = {};
|
|
// for (let i = start; i <= end; i++) {
|
|
// let result = await this.indexPage(i, maxResults);
|
|
// Object.assign(results, result)
|
|
// }
|
|
// return results;
|
|
|
|
// let results = [];
|
|
const indexers= [];
|
|
// let it = 3
|
|
// for (let url of urls) {
|
|
// // let result = await this.indexPage(url, maxResults);
|
|
// // results.push(result);
|
|
// it++
|
|
// indexers.push(new Indexer(url));
|
|
|
|
// }
|
|
|
|
urls.forEach(url => {
|
|
indexers.push(new Indexer(url));
|
|
});
|
|
|
|
return Promise.map(indexers, function (indexer) {
|
|
return indexer.indexWithPagination();
|
|
}).then(async (results) => {
|
|
return results
|
|
})
|
|
|
|
|
|
// let results = [];
|
|
// for (let url of urls) {
|
|
// let result = await this.indexPage(url, maxResults);
|
|
// results.push(result);
|
|
// }
|
|
// return results;
|
|
}
|
|
|
|
async crawl() {
|
|
console.log("OLX CRAWLER: start crawl");
|
|
|
|
const filteredResults = [];
|
|
const realestateRequests = await allRERequest();
|
|
console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
|
|
const urls = this.createRequestUrls(realestateRequests);
|
|
let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
|
|
|
|
console.log(results[0]);
|
|
|
|
// for (const result of results) {
|
|
// for (const finalResult of result) {
|
|
// if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
|
|
// const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
|
|
|
|
// if (pointInsideBoundingBox[0].length !== 0) {
|
|
// filteredResults.push(finalResult);
|
|
// }
|
|
// }
|
|
// }
|
|
// }
|
|
// console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
|
|
// return filteredResults;
|
|
}
|
|
|
|
createRequestUrls(realestateRequests) {
|
|
const urls = []
|
|
|
|
for (const request of realestateRequests) {
|
|
const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
|
|
const region = "kanton=" + getRegion(request.region).olxid;
|
|
const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
|
|
const sizeMin = "kvadrata_min=" + request.sizeMin;
|
|
const sizeMax = "kvadrata_max=" + request.sizeMax;
|
|
const priceMin = "od=" + request.priceMin;
|
|
const priceMax = "do=" + request.priceMax;
|
|
|
|
const olxUrl = {
|
|
url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}&stranica=`,
|
|
email: request.email,
|
|
uuid: request.uniqueId
|
|
}
|
|
console.log(olxUrl.url);
|
|
urls.push(olxUrl);
|
|
}
|
|
|
|
return urls;
|
|
}
|
|
};
|
|
|
|
|
|
class Indexer {
|
|
|
|
/**
|
|
*
|
|
* @param {String|Array} olxUrl single or array of URLS
|
|
* @param {String} email
|
|
* @param {Strung} uuid
|
|
*/
|
|
|
|
constructor(olxUrl, email, uuid) {
|
|
this.olxUrl = olxUrl
|
|
this.email = email
|
|
this.uuid = uuid
|
|
}
|
|
|
|
async indexWithPagination(pageNumber = 1){
|
|
|
|
let definitiveResults = [];
|
|
console.log("This is olxUrl:" + this.olxUrl.url);
|
|
const pageNr = this.olxUrl.url.match(/\d+$/);
|
|
const indexers = this.iteratePages(pageNumber ? [pageNumber] : pageNr);
|
|
console.log("This is page number:" + pageNr);
|
|
// if (pageNr && Number(pageNr[0]) >= 20){
|
|
// console.log("Returning page numbermax");
|
|
// return;
|
|
// }
|
|
|
|
// if (pageNumber && pageNumber >= 30){
|
|
// console.log("Returning page numbermax");
|
|
// return;
|
|
// }
|
|
|
|
try{
|
|
var lastPageNumber = indexers.lastPageNumber;
|
|
|
|
return Promise.map(indexers.indexers, function (indexer) {
|
|
console.log("Page number before async");
|
|
console.log(pageNumber);
|
|
return indexer.indexPage(pageNumber);
|
|
}).then(async (results) => {
|
|
let hasResults = false;
|
|
results.forEach(result => {
|
|
if (!hasResults){
|
|
console.log("No results detected")
|
|
hasResults = result.hasResults
|
|
}
|
|
});
|
|
|
|
|
|
// console.log(results);
|
|
console.log("PageNumber");
|
|
console.log(pageNumber);
|
|
if (!hasResults){
|
|
console.log("HAS NO RESULTS");
|
|
// console.log(results);
|
|
return results
|
|
} else {
|
|
console.log("HAS RESULTS");
|
|
console.log(" Results PageNumber");
|
|
console.log(results[0].pageNumber)
|
|
const newResults = await this.indexWithPagination(results[0].pageNumber + 10);
|
|
// console.log(newResults);
|
|
Array.prototype.push.apply(results,newResults);
|
|
return results;
|
|
// console.log("Idnexing seccond");
|
|
// // this.olxUrl.url = this.olxUrl.url.replace(/\d+$/, "")+lastPageNumber
|
|
// // // return await this.indexWithPagination();
|
|
// // Array.prototype.push.apply(definitiveResults,results);
|
|
// definitiveResults = results;
|
|
// console.log("This is FIRST RESULT array");
|
|
// // console.log(results);
|
|
// console.log("This is definitive array after first results");
|
|
// console.log(definitiveResults);
|
|
// const pageNr2 = [10]
|
|
// const indexers2 = this.iteratePages([pageNr2[0]]);
|
|
// console.log("Indexers 2" + indexers2);
|
|
// console.log( indexers2);
|
|
// // const moreIndexers = this.iteratePages()
|
|
// return Promise.map(indexers2.indexers, function (seccondIndexer) {
|
|
// console.log("Indexing seccond indexPage()");
|
|
// return seccondIndexer.indexPage();
|
|
// }).then(async (results) => {
|
|
// console.log("Indexing seccond results ");
|
|
// Array.prototype.push.apply(definitiveResults,results);
|
|
// console.log("This is SECCOND RESULT array");
|
|
// // console.log(results);
|
|
// console.log("This is definitive array after SECCOND results");
|
|
// console.log(definitiveResults);
|
|
// return definitiveResults;
|
|
// });
|
|
}
|
|
});
|
|
} catch(e) {
|
|
console.error("Error has accured", e);
|
|
}
|
|
|
|
}
|
|
|
|
iteratePages(pageNr) {
|
|
console.log("Entering iterate pages: page nr - " + pageNr );
|
|
const indexers = [];
|
|
let lastPageNumber;
|
|
if (pageNr){
|
|
for (let index = Number(pageNr[0]); index <= Number(pageNr[0]) + 10; index++) {
|
|
lastPageNumber = index;
|
|
|
|
// console.log("width number")
|
|
// console.log("page number: " + Number(pageNr[0]))
|
|
// console.log("index: " + index )
|
|
console.log(this.olxUrl.url.replace(/\d+$/, "")+index)
|
|
indexers.push(new Indexer({url : this.olxUrl.url.replace(/\d+$/, "")+index}))
|
|
|
|
}
|
|
} else {
|
|
for (let index = 1; index <= 10; index++) {
|
|
lastPageNumber = index;
|
|
// console.log("widouth number")
|
|
// console.log("index: " + index )
|
|
// console.log(this.olxUrl.url+index)
|
|
indexers.push(new Indexer({url :this.olxUrl.url+index}))
|
|
}
|
|
}
|
|
return {indexers : indexers, lastPageNumber : lastPageNumber};
|
|
}
|
|
|
|
async indexPage(pageNumber) {
|
|
console.log("Page number in index page")
|
|
console.log(pageNumber);
|
|
|
|
try {
|
|
//TODO fix paging
|
|
// console.log('Starting to index page: ' + pageNr);
|
|
// const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
|
|
console.log("Indexing page: " + this.olxUrl.url);
|
|
const res = await fetch(this.olxUrl.url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
const hrefs = [];
|
|
const results = [];
|
|
let hasResults = false
|
|
|
|
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
|
|
hasResults = true
|
|
const href = $(elem).find('a').first().attr('href');
|
|
hrefs.push(href);
|
|
});
|
|
|
|
// let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
|
|
|
|
// for (let i = 0; i < hrefs.length; i++) {
|
|
// console.log(`indexing: ${hrefs[i]}`);
|
|
|
|
// const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email, this.olxUrl.uuid);
|
|
|
|
// if (singleData) {
|
|
// results.push(singleData);
|
|
// }
|
|
// }
|
|
|
|
// await this.sleep(this.olxUrl);
|
|
// console.log('Finished indexing PAGE');
|
|
// const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
|
|
// return Promise.map(singleIndex, function (indexer) {
|
|
// return indexer.indexSingle();
|
|
// }).then(async (results) => {
|
|
|
|
// return results
|
|
// })
|
|
console.log("this is hrefs for olxUrl" + this.olxUrl.url);
|
|
|
|
console.log("NUMBER OF HREFS " + hrefs.length);
|
|
// console.log("HREFS " + hrefs);
|
|
console.log("HAS NO MORE RESULTS " + hasResults);
|
|
|
|
return {hrefs : hrefs, hasResults : hasResults, pageNumber : pageNumber}
|
|
|
|
// return results;
|
|
} catch (e) {
|
|
console.error('Exception caught:' + e);
|
|
}
|
|
}
|
|
|
|
async indexAllHrefs () {
|
|
|
|
}
|
|
|
|
async indexSingle() {
|
|
// try {
|
|
// const res = await fetch(url);
|
|
// const body = await res.text();
|
|
// const $ = cheerio.load(body);
|
|
|
|
// //TODO figure out what to do with username
|
|
// const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
|
|
|
|
// // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
|
|
// // return null;
|
|
// // }
|
|
|
|
// //TODO remove properties that are not needed, and add some if they are missing
|
|
// const title = $('#naslovartikla').text();
|
|
// const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
|
|
|
|
// const price = $('#pc > p:nth-child(2)').text();
|
|
// const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
|
|
// const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
|
|
// const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
|
|
// const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
|
|
// const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
|
|
|
|
// const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
|
|
// const time = $('time').attr('datetime');
|
|
// const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
|
|
|
|
// const descriptions = $('.artikal_detaljniopis_tekst');
|
|
// // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
|
|
// const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
|
// const imgRe = /href":("[^"]*")/g;
|
|
// const matches = latLngRe.exec(body);
|
|
// let lng = '',
|
|
// lat = '';
|
|
|
|
|
|
// const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
|
|
// const parsePrice = (price) => parseFloat(price.replace(".", ""))
|
|
|
|
|
|
// // TODO we dont save images ??
|
|
|
|
// // const images = [];
|
|
// // const imgMatches = body.match(imgRe);
|
|
|
|
// // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
|
|
// // let img = imgMatches[i].replace("href\":", "")
|
|
// // img = img.replace("\"", "");
|
|
// // img = img.replace("\"", "");
|
|
// // images.push(img);
|
|
// // }
|
|
|
|
// // const uploadPromises = images.map(img => {
|
|
// // const imgFixed = eval(`'${img}'`);
|
|
// // return cloudinary.uploader.upload(eval(`'${img}'`));
|
|
// // });
|
|
|
|
// // const uploadResults = await Promise.all(uploadPromises);
|
|
// // const cloudinaryImages = uploadResults.map(ur => ur.url);
|
|
|
|
// if (matches && matches.length >= 3) {
|
|
// lat = matches[1];
|
|
// lng = matches[2];
|
|
// }
|
|
|
|
// const parsedPrice = parsePrice(price);
|
|
|
|
// const locationArray = location.split(",");
|
|
// const region = locationArray[0];
|
|
// const municipality = locationArray[1];
|
|
|
|
// const data = {
|
|
// realEstateType: this.getCategoryId(realEstateType),
|
|
// email : email,
|
|
// olxId: olxId,
|
|
// // category: category,
|
|
// url,
|
|
// title,
|
|
// price: isNaN(parsedPrice) ? price : parsedPrice,
|
|
// size: parseFloat(size),
|
|
// gardenSize: parseFloat(gardenSize),
|
|
// address,
|
|
// region,
|
|
// municipality,
|
|
// // adType: AD_TYPE_SALE,
|
|
// time,
|
|
// shortDescription: descriptions.first().text(),
|
|
// longDescription: descriptions.last().text(),
|
|
// lat,
|
|
// lng,
|
|
// loc: [parseFloat(lat), parseFloat(lng)],
|
|
// // images: cloudinaryImages
|
|
// };
|
|
|
|
// return data;
|
|
// } catch (e) {
|
|
// console.error('Exception caught: ' + e.message);
|
|
// }
|
|
|
|
// return null;
|
|
await this.sleep(this.olxUrl);
|
|
console.log("Finished indexing single page");
|
|
return {};
|
|
}
|
|
|
|
async sleep(ms) {
|
|
console.log(ms);
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
}
|
|
|