140 lines
3.9 KiB
JavaScript
140 lines
3.9 KiB
JavaScript
'use strict'
|
|
|
|
let fetch = require('node-fetch');
|
|
let cheerio = require('cheerio');
|
|
let fs = require('fs');
|
|
|
|
|
|
const pagesToTake = 10;
|
|
|
|
export default class OlxCrawler {
|
|
|
|
constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
|
|
this.fromPage = fromPage;
|
|
this.toPage = toPage;
|
|
this.maxResults = maxResults;
|
|
}
|
|
|
|
async indexSingle(url) {
|
|
try {
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
|
|
const title = $('#naslovartikla').text();
|
|
const price = $('#pc > p:nth-child(2)').text();
|
|
const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
|
|
const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
|
|
const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
|
|
const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
|
|
|
|
const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
|
|
const time = $('time').attr('datetime');
|
|
const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
|
|
|
|
const descriptions = $('.artikal_detaljniopis_tekst');
|
|
const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
|
|
const imgRe = /href":("[^"]*")/g;
|
|
const matches = latLngRe.exec(body);
|
|
let lng = '',
|
|
lat = '';
|
|
|
|
const images = [];
|
|
const imgMatches = body.match(imgRe);
|
|
|
|
for (let i = 0; imgMatches && i < imgMatches.length; i++) {
|
|
let img = imgMatches[i].replace("href\":", "")
|
|
img = img.replace("\"", "");
|
|
img = img.replace("\"", "");
|
|
images.push(img);
|
|
}
|
|
|
|
if (matches && matches.length >= 3) {
|
|
lat = matches[1];
|
|
lng = matches[2];
|
|
}
|
|
|
|
const data = {
|
|
url,
|
|
title,
|
|
price,
|
|
size,
|
|
rooms,
|
|
address,
|
|
location,
|
|
adType,
|
|
time,
|
|
olxId,
|
|
shortDescription: descriptions.first().text(),
|
|
longDescription: descriptions.last().text(),
|
|
lat,
|
|
lng,
|
|
images
|
|
};
|
|
|
|
return data;
|
|
} catch (e) {
|
|
console.error('Exception caught: ' + e);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async indexPage(pageNr, maxResults = 1000) {
|
|
try {
|
|
console.log('Starting to index page: ' + pageNr);
|
|
const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
|
|
|
|
const res = await fetch(url);
|
|
const body = await res.text();
|
|
const $ = cheerio.load(body);
|
|
const hrefs = [];
|
|
const results = {};
|
|
|
|
$('#rezultatipretrage').find('.listitem').each((i, elem) => {
|
|
const href = $(elem).find('a').first().attr('href');
|
|
hrefs.push(href);
|
|
});
|
|
|
|
let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
|
|
|
|
for (let i = 0; i < actualNoOfResults; i++) {
|
|
console.log(`indexing: ${hrefs[i]}`);
|
|
|
|
const singleData = await this.indexSingle(hrefs[i]);
|
|
|
|
if (singleData) {
|
|
results[hrefs[i]] = singleData;
|
|
}
|
|
await this.sleep(500);
|
|
}
|
|
|
|
return results;
|
|
} catch (e) {
|
|
console.error('Exception caught:' + e);
|
|
}
|
|
}
|
|
|
|
async sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async indexPages(start, end, maxResults = 1000) {
|
|
let results = {};
|
|
for (let i = start; i <= end; i++) {
|
|
let result = await this.indexPage(i, numberOfResults);
|
|
Object.assign(results, result)
|
|
await this.sleep(5000);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async crawl() {
|
|
let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
|
|
return results;
|
|
}
|
|
}
|
|
|
|
|
|
//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');
|