Correctly parse rooms, size, price etc.

crawler/specific/olx.js

@@ -3,6 +3,7 @@
 let fetch = require('node-fetch');
 let cheerio = require('cheerio');
 let fs = require('fs');
+import {AD_TYPE_SALE, IGNORED_USERNAMES} from '../enums';
 
 export default class OlxCrawler {
 
@@ -18,6 +19,12 @@ export default class OlxCrawler {
       const body = await res.text();
       const $ = cheerio.load(body);
 
+      const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+      if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+        return null;
+      }
+
       const title = $('#naslovartikla').text();
       const price = $('#pc > p:nth-child(2)').text();
       const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
@@ -30,6 +37,7 @@ export default class OlxCrawler {
       const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
 
       const descriptions = $('.artikal_detaljniopis_tekst');
+      const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
       const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
       const imgRe = /href":("[^"]*")/g;
       const matches = latLngRe.exec(body);
@@ -39,6 +47,10 @@ export default class OlxCrawler {
       const images = [];
       const imgMatches = body.match(imgRe);
 
+      const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+      const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
       for (let i = 0; imgMatches && i < imgMatches.length; i++) {
         let img = imgMatches[i].replace("href\":", "")
         img = img.replace("\"", "");
@@ -54,14 +66,14 @@ export default class OlxCrawler {
       const data = {
         url,
         title,
-        price,
-        size,
-        rooms,
+        price: parsePrice(price) || -1,
+        size: parseFloat(size),
+        rooms: parseRooms(rooms),
+        floor: parseInt(floor),
         address,
         location,
-        adType,
+        adType: AD_TYPE_SALE,
         time,
-        olxId,
         shortDescription: descriptions.first().text(),
         longDescription: descriptions.last().text(),
         lat,
@@ -81,7 +93,7 @@ export default class OlxCrawler {
   async indexPage(pageNr, maxResults = 1000) {
     try {
       console.log('Starting to index page: ' + pageNr);
-      const url = `http://www.olx.ba/pretraga?vrsta=samoizdavanje&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+      const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
 
       const res     = await fetch(url);
       const body    = await res.text();