diff --git a/app/helpers/crawlers/olixClawler.js b/app/helpers/crawlers/olixClawler.js index 1c37df9..0b9b01c 100644 --- a/app/helpers/crawlers/olixClawler.js +++ b/app/helpers/crawlers/olixClawler.js @@ -3,28 +3,161 @@ const cheerio = require('cheerio'); export default class OlxCrawler { - constructor(fromPage = 0, toPage = 10, maxResults = 1000) { - this.fromPage = fromPage; - this.toPage = toPage; - this.maxResults = maxResults; + constructor(fromPage = 0, toPage = 10, maxResults = 1000) { + this.fromPage = fromPage; + this.toPage = toPage; + this.maxResults = maxResults; + } + + async indexSingle(url) { + try { + const res = await fetch(url); + const body = await res.text(); + const $ = cheerio.load(body); + + const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text(); + + if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) { + return null; } - async sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); + const title = $('#naslovartikla').text(); + const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text(); + + const price = $('#pc > p:nth-child(2)').text(); + const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text(); + const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text(); + const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text(); + const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content'); + + const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text(); + const time = $('time').attr('datetime'); + const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text(); + + const descriptions = $('.artikal_detaljniopis_tekst'); + const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text(); + const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g; + const imgRe = /href":("[^"]*")/g; + const matches = latLngRe.exec(body); + let lng = '', + lat = ''; + + const images = []; + const imgMatches = body.match(imgRe); + + const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join()) + const parsePrice = (price) => parseFloat(price.replace(".", "")) + + + for (let i = 0; imgMatches && i < imgMatches.length; i++) { + let img = imgMatches[i].replace("href\":", "") + img = img.replace("\"", ""); + img = img.replace("\"", ""); + images.push(img); } - async indexPages(start, end, maxResults = 1000) { - let results = {}; - for (let i = start; i <= end; i++) { - let result = await this.indexPage(i, maxResults); - Object.assign(results, result) - await this.sleep(5000); + const uploadPromises = images.map(img => { + const imgFixed = eval(`'${img}'`); + return cloudinary.uploader.upload(eval(`'${img}'`)); + }); + + const uploadResults = await Promise.all(uploadPromises); + const cloudinaryImages = uploadResults.map(ur => ur.url); + + if (matches && matches.length >= 3) { + lat = matches[1]; + lng = matches[2]; + } + + const parsedPrice = parsePrice(price); + let parsedRooms; + + if (rooms === 'Garsonjera') { + parsedRooms = 0; + } else { + parsedRooms = parseRooms(rooms); + } + + const data = { + category: this.getCategoryId(category), + url, + title, + price: isNaN(parsedPrice) ? price : parsedPrice, + size: parseFloat(size), + rooms: parsedRooms, + floor: parseInt(floor), + address, + location, + adType: AD_TYPE_SALE, + time, + shortDescription: descriptions.first().text(), + longDescription: descriptions.last().text(), + lat, + lng, + loc: [parseFloat(lat), parseFloat(lng)], + images: cloudinaryImages + }; + + return data; + } catch (e) { + console.error('Exception caught: ' + e.message); + } + + return null; + } + + async indexPage(url, pageNr, maxResults = 1000) { + try { + console.log('Starting to index page: ' + pageNr); + const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`; + + const res = await fetch(url); + const body = await res.text(); + const $ = cheerio.load(body); + const hrefs = []; + const results = {}; + + $('#rezultatipretrage').find('.listitem').each((i, elem) => { + const href = $(elem).find('a').first().attr('href'); + hrefs.push(href); + }); + + let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults; + + for (let i = 0; i < hrefs.length; i++) { + console.log(`indexing: ${hrefs[i]}`); + + const singleData = await this.indexSingle(hrefs[i]); + + if (singleData) { + results[hrefs[i]] = singleData; } - return results; - } - - async crawl() { - let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults); - return results; + await this.sleep(500); } + + return results; + } catch (e) { + console.error('Exception caught:' + e); + } + } + + async sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + async indexPages(url, start, end, maxResults = 1000) { + let results = {}; + for (let i = start; i <= end; i++) { + let result = await this.indexPage(i, maxResults); + Object.assign(results, result) + await this.sleep(5000); + } + return results; + } + + async crawl() { + // TODO create URLS from db + let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults); + return results; + } } \ No newline at end of file diff --git a/app/helpers/enums.js b/app/helpers/enums.js index 2c3b177..9ab2d58 100644 --- a/app/helpers/enums.js +++ b/app/helpers/enums.js @@ -1,7 +1,7 @@ const realEstateTypes = [ - { title: "Kuća", id: "kuca", hasGardenSize: true }, - { title: "Stan", id: "stan", hasGardenSize: false }, - { title: "Vikendica", id: "vikendica", hasGardenSize: true } + { title: "Kuća", id: "kuca", hasGardenSize: true, olixCategory: 23 }, + { title: "Stan", id: "stan", hasGardenSize: false, olixCategory: 24}, + { title: "Vikendica", id: "vikendica", hasGardenSize: true, olixCategory: 26 } ]; const sizes = [ diff --git a/app/services/crawlerService.js b/app/services/crawlerService.js index 073195c..c6f0c82 100644 --- a/app/services/crawlerService.js +++ b/app/services/crawlerService.js @@ -1,5 +1,6 @@ +var http = require('http'); const crawlers = [ //new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS), ]; diff --git a/package.json b/package.json index 8196722..a8b5aa4 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "test": "echo \"Error: no test specified\" && exit 1", "start": "node ./index.js", "start-mon": "nodemon ./index.js", + "scheduler": "node ./app/services/crawlerService.js", "migrate": "cd app && npx sequelize db:migrate", "setup": "docker build -t marketalerts . && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate", "docker-start": "docker start pg_marketalerts",