old-kivi/crawler/specific/prostor.js

'use strict';

let fetch = require ('node-fetch');
let cheerio = require ('cheerio');
let fs = require ('fs');
let cloudinary = require ('cloudinary');
let FormData = require ('form-data');

import {
  AD_TYPE_SALE,
  IGNORED_USERNAMES,
  CATEGORY_FLAT,
  CATEGORY_HOUSE,
  CATEGORY_OFFICE,
  CATEGORY_LAND,
  STATUS_NORMAL,
  STATUS_RESERVED,
  STATUS_SOLD,
} from '../../common/enums';

export default class ProstorCrawler {
  constructor (fromPage = 0, toPage = 10, maxResults = 1000) {
    this.fromPage = fromPage;
    this.toPage = toPage;
    this.maxResults = maxResults;
  }

  async indexSingle (url) {
    try {
      const res = await fetch (url);
      const body = await res.text ();
      const $ = cheerio.load (body);

      const title = $ (
        '#nav_center_sub > div.content_area_1_left > div:nth-child(1) > h1'
      ).text ();

      const category = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(3) > div:nth-child(4) > div.size_rs > span'
      ).text ();

      const price = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(1) > div.size_rs > strong'
      ).text ();

      const size = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(7) > div.size_rs > span'
      ).text ();
      const rooms = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(2) > div.size_rs > span'
      ).text ();

      const address = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(3) > div:nth-child(3) > div.size_rs > span'
      ).text ();

      //const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');

      //const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();

      const time = $ (
        '#nav_center_sub > div.content_area_1_right > div.bottom_d > div > strong:nth-child(1)'
      ).text ();

      //const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();

      const descriptions = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_ll_in_show > div:nth-child(1)'
      ).text ();

      const floor = $ (
        '#nav_center_sub > div.content_area_1_left > div.bottom10 > div.content_lr_in_show > div:nth-child(4) > div:nth-child(6) > div.size_rs'
      ).text ();

      const latLngRe = /marker=([0-9]+\.[0-9]+)\,\s*([0-9]+\.[0-9]+)/g;

      var hasMap = false;

      var tmpTitle = title.toUpperCase ();

      var status = STATUS_NORMAL;
      if (tmpTitle.indexOf ('PRODANO') !== -1) status = STATUS_SOLD;
      if (tmpTitle.indexOf ('REZERVISANO') !== -1) status = STATUS_RESERVED;

      //const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;

      const matches = latLngRe.exec (body);
      let lng = '', lat = '';
      hasMap = false;
      if (matches && matches.length >= 3) {
        lat = matches[1];
        lng = matches[2];
        hasMap = true;
      }

      //console.log({
      //lat,
      //lng,
      //floor,
      //descriptions,
      //time,
      //price,
      //size,
      //category,
      //title
      //});

      //const imgRe = /href":("[^"]*")/g;

      const images = [];

      //const imgMatches = body.match(imgRe);

      const parseRooms = rooms =>
        parseInt (
          [...rooms].filter (c => !isNaN (c)).filter (c => c.trim ()).join ()
        );
      const parsePrice = price => parseFloat (price.replace ('.', ''));

      $ ('.fancybox').each ((i, elem) => {
        const img = $ (elem).attr ('href');
        images.push (img);
      });

      //for (let i = 0; imgMatches && i < imgMatches.length; i++) {
      //let img = imgMatches[i].replace("href\":", "")
      //img = img.replace("\"", "");
      //img = img.replace("\"", "");
      //images.push(img);
      //}

      //const uploadPromises = images.map(img => {
      //return cloudinary.uploader.upload(img);
      //});

      //const uploadResults = await Promise.all(uploadPromises);
      //const cloudinaryImages = uploadResults.map(ur => ur.url);

      const parsedPrice = parsePrice (price);
      let parsedRooms;

      if (rooms === 'Garsonjera') {
        parsedRooms = 0;
      } else {
        parsedRooms = parseRooms (rooms);
      }

      const data = {
        category: this.getCategoryId (category),
        url,
        title,
        price: isNaN (parsedPrice) ? price : parsedPrice,
        size: parseFloat (size),
        rooms: parsedRooms,
        floor: parseInt (floor),
        address,
        adType: AD_TYPE_SALE,
        time,
        shortDescription: title,
        longDescription: descriptions,
        lat,
        lng,
        loc: [parseFloat (lat), parseFloat (lng)],
        hasMap,
        status,
        //images: cloudinaryImages
        images,
      };
      console.log (data);

      return data;
    } catch (e) {
      console.error ('Exception caught: ' + e.message);
    }

    return null;
  }

  async indexPage (pageNr, maxResults = 1000) {
    try {
      console.log ('Starting to index page: ' + pageNr);
      const url = `http://prostor.ba/index.php`;

      const data = new FormData ();
      data.append ('sortCombo', 'e.date_create DESC');
      data.append ('command', '');
      data.append ('action', 'show');
      data.append ('page', pageNr);
      data.append ('param', 'ponuda.inc.php');
      data.append ('checkNO', 0);
      data.append ('order', 'e.date_create DESC');
      data.append ('reset', 0);
      data.append ('estate_action', 1);
      data.append ('Itemid', 785);

      const res = await fetch (url, {
        method: 'POST',
        body: data,
      });
      const body = await res.text ();
      const $ = cheerio.load (body);

      const hrefs = [];
      $ ('.nekret_box').each ((i, elem) => {
        const href = $ (elem).find ('a').first ().attr ('href');
        hrefs.push (`http://prostor.ba/${href}`);
      });

      const results = {};
      for (const href of hrefs) {
        console.log (`indexing: ${href}`);

        const singleData = await this.indexSingle (href);

        if (singleData) {
          results[href] = singleData;
        }

        await this.sleep (500);
      }

      return results;
    } catch (e) {
      console.error ('Exception caught:' + e);
    }
  }

  getCategoryId (category) {
    if (category === 'Stan') {
      return CATEGORY_FLAT;
    } else if (category === 'Zemljište') {
      return CATEGORY_LAND;
    } else if (category === 'Kuća') {
      return CATEGORY_HOUSE;
    } else if (category === 'Poslovni prostor') {
      return CATEGORY_OFFICE;
    }
  }

  async sleep (ms) {
    return new Promise (resolve => setTimeout (resolve, ms));
  }

  async indexPages (start, end, maxResults = 1000) {
    let results = {};
    for (let i = start; i <= end; i++) {
      let result = await this.indexPage (i, maxResults);
      Object.assign (results, result);
      await this.sleep (5000);
    }
    return results;
  }

  async crawl () {
    let results = await this.indexPages (
      this.fromPage,
      this.toPage,
      this.maxResults
    );
    return results;
  }
}