old-kivi/crawler/build/olx.js

'use strict';

let indexSingle = (() => {
  var _ref = _asyncToGenerator(function* (url) {
    try {
      const res = yield fetch(url);
      const body = yield res.text();
      const $ = cheerio.load(body);

      const title = $('#naslovartikla').text();
      const price = $('#pc > p:nth-child(2)').text();
      const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
      const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
      const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
      const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');

      const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
      const time = $('time').attr('datetime');
      const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();

      const descriptions = $('.artikal_detaljniopis_tekst');
      const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
      const imgRe = /href":("[^"]*")/g;
      //href":("[^"]*")]")"
      const matches = latLngRe.exec(body);
      let lng = '',
          lat = '';

      const images = [];
      const imgMatches = body.match(imgRe); //imgRe.exec(body);

      console.log(imgMatches);
      for (let i = 0; imgMatches && i < imgMatches.length; i++) {
        let img = imgMatches[i].replace("href\":", "");
        img = img.replace("\"", "");
        img = img.replace("\"", "");
        images.push(img);
      }

      if (matches && matches.length >= 3) {
        lat = matches[1];
        lng = matches[2];
      }

      const data = {
        title,
        price,
        size,
        rooms,
        address,
        location,
        adType,
        time,
        olxId,
        shortDescription: descriptions.first().text(),
        longDescription: descriptions.last().text(),
        lat,
        lng,
        images
      };

      console.log(data);
      return data;
    } catch (e) {
      console.error('Exception caught: ' + e);
    }

    return null;
  });

  return function indexSingle(_x) {
    return _ref.apply(this, arguments);
  };
})();

let indexPage = (() => {
  var _ref2 = _asyncToGenerator(function* (pageNr) {
    try {
      console.log('Starting to index page: ' + pageNr);
      const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${ pageNr }`;

      const res = yield fetch(url);
      const body = yield res.text();
      const $ = cheerio.load(body);
      const hrefs = [];
      const results = {};

      $('#rezultatipretrage').find('.listitem').each(function (i, elem) {
        const href = $(elem).find('a').first().attr('href');
        hrefs.push(href);
      });

      console.log('number to index: ' + hrefs.length);
      for (let i = 0; i < hrefs.length; i++) {
        console.log(`indexing: ${ hrefs[i] }`);

        const singleData = yield indexSingle(hrefs[i]);

        if (singleData) {
          results[hrefs[i]] = singleData;
        }
        yield sleep(500);
      }

      jsonfile.writeFileSync(`izdavanje-sarajevo-page-${ pageNr }.json`, results);
    } catch (e) {
      console.error('Exception caught:' + e);
    }
  });

  return function indexPage(_x2) {
    return _ref2.apply(this, arguments);
  };
})();

let indexPages = (() => {
  var _ref3 = _asyncToGenerator(function* (start, end) {
    for (let i = start; i <= end; i++) {
      console.log('Start page: ', i);
      yield indexPage(i);
      console.log('Done with page!');
      sleep(5000);
    }
  });

  return function indexPages(_x3, _x4) {
    return _ref3.apply(this, arguments);
  };
})();

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }

let fetch = require('node-fetch');
let jsonfile = require('jsonfile');
let cheerio = require('cheerio');
let fs = require('fs');

const pagesToTake = 10;

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

indexPages(8, 10);

//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');