Prostor.ba crawler

crawler/crawl.js

@@ -13,13 +13,16 @@ import {
 } from 'source-map-support';
 import 'dotenv/config';
 import OlxCrawler from './specific/olx';
+import ProstorCrawler from './specific/prostor';
 import MongoSaver from './savers/mongo'
 
 install(); // for source maps to work
 
 let crawlers = [
-  new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS)
+  //new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
+  new ProstorCrawler(process.env.PROSTOR_FROM_PAGE, process.env.PROSTOR_TO_PAGE, process.env.PROSTOR_MAX_RESULTS)
 ];
+
 let savers = [
   new MongoSaver(process.env.MONGO_URL)
 ];

crawler/package.json

@@ -14,6 +14,7 @@
     "cloudinary": "^1.8.0",
     "dotenv": "^2.0.0",
     "fetch": "^1.1.0",
+    "form-data": "^2.1.4",
     "json-loader": "^0.5.4",
     "mongodb": "^2.2.11",
     "node-fetch": "^1.6.3",

crawler/specific/prostor.js

@@ -0,0 +1,234 @@
+'use strict'
+
+let fetch = require('node-fetch');
+let cheerio = require('cheerio');
+let fs = require('fs');
+let cloudinary = require('cloudinary');
+let FormData = require('form-data');
+
+import {
+  AD_TYPE_SALE,
+  IGNORED_USERNAMES,
+  CATEGORY_FLAT,
+  CATEGORY_HOUSE,
+  CATEGORY_OFFICE,
+  CATEGORY_LAND
+} from '../enums';
+
+export default class ProstorCrawler {
+
+  constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+    this.fromPage = fromPage;
+    this.toPage = toPage;
+    this.maxResults = maxResults;
+  }
+
+  async indexSingle(url) {
+    try {
+
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      console.log('bailing...');
+
+      const title = $('#nav_center_sub > div.content_area_1_left > div:nth-child(1) > h1').text();
+
+      const category = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(3) > div:nth-child(4) > div.size_rs > span').text();
+
+
+      const price = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(1) > div.size_rs > strong').text();
+
+      const size = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(7) > div.size_rs > span').text();
+      const rooms = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(2) > div.size_rs > span').text();
+
+      const address = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(3) > div:nth-child(3) > div.size_rs > span').text();
+
+      //const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+      //const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+
+      const time = $('#nav_center_sub > div.content_area_1_right > div.bottom_d > div > strong:nth-child(1)').text();
+
+      //const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+      const descriptions = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_ll_in_show > div:nth-child(1)').text();
+
+
+      const floor = $('#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(6) > div.size_rs').text();
+
+
+      const latLngRe = /marker=([0-9]+\.[0-9]+)\,\s*([0-9]+\.[0-9]+)/g;
+
+      //const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+
+      const matches = latLngRe.exec(body);
+      let lng = '',
+        lat = '';
+      if (matches && matches.length >= 3) {
+        lat = matches[1];
+        lng = matches[2];
+      }
+
+      //console.log({
+        //lat,
+        //lng,
+        //floor,
+        //descriptions,
+        //time,
+        //price,
+        //size,
+        //category,
+        //title
+      //});
+
+      //const imgRe = /href":("[^"]*")/g;
+
+      const images = [];
+
+
+      //const imgMatches = body.match(imgRe);
+
+      const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+      const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+      $('.fancybox').each((i, elem) => {
+        const img = $(elem).attr('href');
+        images.push(img);
+      });
+
+      //for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        //let img = imgMatches[i].replace("href\":", "")
+        //img = img.replace("\"", "");
+        //img = img.replace("\"", "");
+        //images.push(img);
+      //}
+
+      const uploadPromises = images.map(img => {
+        return cloudinary.uploader.upload(img);
+      });
+
+      const uploadResults = await Promise.all(uploadPromises);
+      const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+
+      const parsedPrice = parsePrice(price);
+      let parsedRooms;
+
+      if (rooms === 'Garsonjera') {
+        parsedRooms = 0;
+      } else {
+        parsedRooms = parseRooms(rooms);
+      }
+
+      const data = {
+        category: this.getCategoryId(category),
+        url,
+        title,
+        price: isNaN(parsedPrice) ? price : parsedPrice,
+        size: parseFloat(size),
+        rooms: parsedRooms,
+        floor: parseInt(floor),
+        address,
+        adType: AD_TYPE_SALE,
+        time,
+        shortDescription: title,
+        longDescription: descriptions,
+        lat,
+        lng,
+        loc: [parseFloat(lat), parseFloat(lng)],
+        images: cloudinaryImages
+      };
+      console.log(data);
+
+      return data;
+    } catch (e) {
+      console.error('Exception caught: ' + e.message);
+    }
+
+    return null;
+  }
+
+  async indexPage(pageNr, maxResults = 1000) {
+    try {
+
+      console.log('Starting to index page: ' + pageNr);
+      const url = `http://prostor.ba/index.php`;
+
+      const data = new FormData();
+      data.append('sortCombo', 'e.date_create DESC');
+      data.append('command', '');
+      data.append('action', 'show');
+      data.append('page', pageNr);
+      data.append('param', 'ponuda.inc.php');
+      data.append('checkNO', 0);
+      data.append('order', 'e.date_create DESC');
+      data.append('reset', 0);
+      data.append('estate_action', 1);
+      data.append('Itemid', 785);
+
+      const res = await fetch(url, {
+        method: 'POST',
+        body: data
+      });
+      const body    = await res.text();
+      const $       = cheerio.load(body);
+
+      const hrefs = [];
+      $('.nekret_box').each((i, elem) => {
+        const href = $(elem).find("a").first().attr('href');
+        hrefs.push(`http://prostor.ba/${href}`);
+      });
+
+      const results = {};
+      for (const href of hrefs) {
+        console.log(`indexing: ${href}`);
+
+        const singleData = await this.indexSingle(href);
+
+        if (singleData) {
+          results[href] = singleData;
+        }
+
+        await this.sleep(500);
+      }
+
+      return results;
+    } catch (e) {
+      console.error('Exception caught:' + e);
+    }
+  }
+
+  getCategoryId (category) {
+    if (category === 'Stan') {
+      return CATEGORY_FLAT;
+    } else if (category === 'Zemljište') {
+      return CATEGORY_LAND;
+    } else if (category === 'Kuća') {
+      return CATEGORY_HOUSE;
+    } else if (category === 'Poslovni prostor') {
+      return CATEGORY_OFFICE;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async indexPages(start, end, maxResults = 1000) {
+    let results = {};
+    for (let i = start; i <= end; i++) {
+      let result = await this.indexPage(i, maxResults);
+      Object.assign(results, result)
+      await this.sleep(5000);
+    }
+    return results;
+  }
+
+  async crawl() {
+    let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
+    return results;
+  }
+}
+

crawler/yarn.lock

@@ -1053,6 +1053,14 @@ forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
 
+form-data:
+  version "2.1.4"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1"
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.5"
+    mime-types "^2.1.12"
+
 form-data@~1.0.0-rc4:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-1.0.1.tgz#ae315db9a4907fa065502304a66d7733475ee37c"

web/dist/main.css

@@ -644,6 +644,7 @@ html {
   text-align: center;
   position: relative;
   user-select: none;
+  cursor: pointer;
 }
 
 .ld-image-container .prev-button {