crawlers and savers

crawler/specific/olx.js

@@ -0,0 +1,139 @@
+'use strict'
+
+let fetch = require('node-fetch');
+let cheerio = require('cheerio');
+let fs = require('fs');
+
+
+const pagesToTake = 10;
+
+export default class OlxCrawler {
+
+  constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+    this.fromPage = fromPage;
+    this.toPage = toPage;
+    this.maxResults = maxResults;
+  }
+
+  async indexSingle(url) {
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      const title = $('#naslovartikla').text();
+      const price = $('#pc > p:nth-child(2)').text();
+      const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+      const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+      const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+      const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+      const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+      const time = $('time').attr('datetime');
+      const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+      const descriptions = $('.artikal_detaljniopis_tekst');
+      const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+      const imgRe = /href":("[^"]*")/g;
+      const matches = latLngRe.exec(body);
+      let lng = '',
+        lat = '';
+
+      const images = [];
+      const imgMatches = body.match(imgRe);
+
+      for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        let img = imgMatches[i].replace("href\":", "")
+        img = img.replace("\"", "");
+        img = img.replace("\"", "");
+        images.push(img);
+      }
+
+      if (matches && matches.length >= 3) {
+        lat = matches[1];
+        lng = matches[2];
+      }
+
+      const data = {
+        url,
+        title,
+        price,
+        size,
+        rooms,
+        address,
+        location,
+        adType,
+        time,
+        olxId,
+        shortDescription: descriptions.first().text(),
+        longDescription: descriptions.last().text(),
+        lat,
+        lng,
+        images
+      };
+
+      return data;
+    } catch (e) {
+      console.error('Exception caught: ' + e);
+    }
+
+    return null;
+  }
+
+  async indexPage(pageNr, maxResults = 1000) {
+    try {
+      console.log('Starting to index page: ' + pageNr);
+      const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      const hrefs = [];
+      const results = {};
+
+      $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+        const href = $(elem).find('a').first().attr('href');
+        hrefs.push(href);
+      });
+
+      let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+      for (let i = 0; i < actualNoOfResults; i++) {
+        console.log(`indexing: ${hrefs[i]}`);
+
+        const singleData = await this.indexSingle(hrefs[i]);
+
+        if (singleData) {
+          results[hrefs[i]] = singleData;
+        }
+        await this.sleep(500);
+      }
+
+      return results;
+    } catch (e) {
+      console.error('Exception caught:' + e);
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async indexPages(start, end, maxResults = 1000) {
+    let results = {};
+    for (let i = start; i <= end; i++) {
+      let result = await this.indexPage(i, numberOfResults);
+      Object.assign(results, result)
+      await this.sleep(5000);
+    }
+    return results;
+  }
+
+  async crawl() {
+    let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
+    return results;
+  }
+}
+
+
+//indexSingle('http://www.olx.ba/artikal/23198642/trosoban-stan-centar-josipa-vancasa/');