old-kivi/crawler/olx.js

'use strict'

let fetch    = require('node-fetch');
let jsonfile = require('jsonfile');
let cheerio  = require('cheerio');
let fs       = require('fs');

const pagesToTake = 10;

async function indexSingle(url) {
  try {
    const res   = await fetch(url);
    const body  = await res.text();
    const $     = cheerio.load(body);

    const title    = $('#naslovartikla').text();
    const price    = $('#pc > p:nth-child(2)').text();
    const size     = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
    const rooms    = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
    const address  = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
    const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');

    const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
    const time   = $('time').attr('datetime');
    const olxId  = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();

    const descriptions = $('.artikal_detaljniopis_tekst');
    const latLngRe     = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
    const imgRe        = /href":("[^"]*")/g;
    const matches      = latLngRe.exec(body);
    let lng            = '',
      lat              = '';

    const images = [];
    const imgMatches = body.match(imgRe);

    console.log(imgMatches);
    for(let i = 0; imgMatches && i < imgMatches.length; i++) {
      let img = imgMatches[i].replace("href\":", "")
      img     = img.replace("\"", "");
      img     = img.replace("\"", "");
      images.push(img);
    }

    if (matches && matches.length >= 3) {
      lat = matches[1];
      lng = matches[2];
    }

      const data = {
        title,
        price,
        size,
        rooms,
        address,
        location,
        adType,
        time,
        olxId,
        shortDescription: descriptions.first().text(),
        longDescription: descriptions.last().text(),
        lat,
        lng,
        images
      };

      return data;
  } catch(e) {
    console.error('Exception caught: ' + e);
  }

  return null;
}



async function indexPage(pageNr) {
  try {
    console.log('Starting to index page: ' + pageNr);
    const url   = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;

    const res     = await fetch(url);
    const body    = await res.text();
    const $       = cheerio.load(body);
    const hrefs   = [];
    const results = {};

    $('#rezultatipretrage').find('.listitem').each((i, elem) => {
      const href = $(elem).find('a').first().attr('href');
      hrefs.push(href);
    });

    for(let i = 0; i < hrefs.length; i++) {
      console.log(`indexing: ${hrefs[i]}`);

      const singleData  = await indexSingle(hrefs[i]);

      if (singleData) {
        results[hrefs[i]] = singleData;
      }
      await sleep(500);
    }

    jsonfile.writeFileSync(`izdavanje-sarajevo-page-${pageNr}.json`, results);
  } catch(e) {
    console.error('Exception caught:' + e);
  }
}


function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

async function indexPages(start, end) {
  for(let i = start; i <= end; i++) {
    await indexPage(i);
    sleep(5000);
  }
}

indexPages(8, 10);

//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');