Scheduler

app/helpers/crawlers/olixClawler.js

@@ -3,28 +3,161 @@ const cheerio = require('cheerio');
 
 export default class OlxCrawler {
 
-    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
-        this.fromPage = fromPage;
-        this.toPage = toPage;
-        this.maxResults = maxResults;
+  constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+    this.fromPage = fromPage;
+    this.toPage = toPage;
+    this.maxResults = maxResults;
+  }
+
+  async indexSingle(url) {
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+      if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+        return null;
       }
 
-      async sleep(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
+      const title = $('#naslovartikla').text();
+      const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+      const price = $('#pc > p:nth-child(2)').text();
+      const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+      const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+      const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+      const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+      const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+      const time = $('time').attr('datetime');
+      const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+      const descriptions = $('.artikal_detaljniopis_tekst');
+      const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+      const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+      const imgRe = /href":("[^"]*")/g;
+      const matches = latLngRe.exec(body);
+      let lng = '',
+        lat = '';
+
+      const images = [];
+      const imgMatches = body.match(imgRe);
+
+      const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+      const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+      for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        let img = imgMatches[i].replace("href\":", "")
+        img = img.replace("\"", "");
+        img = img.replace("\"", "");
+        images.push(img);
       }
 
-      async indexPages(start, end, maxResults = 1000) {
-        let results = {};
-        for (let i = start; i <= end; i++) {
-          let result = await this.indexPage(i, maxResults);
-          Object.assign(results, result)
-          await this.sleep(5000);
+      const uploadPromises = images.map(img => {
+        const imgFixed = eval(`'${img}'`);
+        return cloudinary.uploader.upload(eval(`'${img}'`));
+      });
+
+      const uploadResults = await Promise.all(uploadPromises);
+      const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+      if (matches && matches.length >= 3) {
+        lat = matches[1];
+        lng = matches[2];
+      }
+
+      const parsedPrice = parsePrice(price);
+      let parsedRooms;
+
+      if (rooms === 'Garsonjera') {
+        parsedRooms = 0;
+      } else {
+        parsedRooms = parseRooms(rooms);
+      }
+
+      const data = {
+        category: this.getCategoryId(category),
+        url,
+        title,
+        price: isNaN(parsedPrice) ? price : parsedPrice,
+        size: parseFloat(size),
+        rooms: parsedRooms,
+        floor: parseInt(floor),
+        address,
+        location,
+        adType: AD_TYPE_SALE,
+        time,
+        shortDescription: descriptions.first().text(),
+        longDescription: descriptions.last().text(),
+        lat,
+        lng,
+        loc: [parseFloat(lat), parseFloat(lng)],
+        images: cloudinaryImages
+      };
+
+      return data;
+    } catch (e) {
+      console.error('Exception caught: ' + e.message);
+    }
+
+    return null;
+  }
+
+  async indexPage(url, pageNr, maxResults = 1000) {
+    try {
+      console.log('Starting to index page: ' + pageNr);
+      const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      const hrefs = [];
+      const results = {};
+
+      $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+        const href = $(elem).find('a').first().attr('href');
+        hrefs.push(href);
+      });
+
+      let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+      for (let i = 0; i < hrefs.length; i++) {
+        console.log(`indexing: ${hrefs[i]}`);
+
+        const singleData = await this.indexSingle(hrefs[i]);
+
+        if (singleData) {
+          results[hrefs[i]] = singleData;
         }
-        return results;
-      }
-      
-      async crawl() {
-        let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
-        return results;
+        await this.sleep(500);
       }
+
+      return results;
+    } catch (e) {
+      console.error('Exception caught:' + e);
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async indexPages(url, start, end, maxResults = 1000) {
+    let results = {};
+    for (let i = start; i <= end; i++) {
+      let result = await this.indexPage(i, maxResults);
+      Object.assign(results, result)
+      await this.sleep(5000);
+    }
+    return results;
+  }
+
+  async crawl() {
+    // TODO create URLS from db
+    let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
+    return results;
+  }
 }

app/helpers/enums.js

@@ -1,7 +1,7 @@
 const realEstateTypes = [
-    { title: "Kuća", id: "kuca", hasGardenSize: true },
-    { title: "Stan", id: "stan", hasGardenSize: false },
-    { title: "Vikendica", id: "vikendica", hasGardenSize: true }
+    { title: "Kuća", id: "kuca", hasGardenSize: true, olixCategory: 23 },
+    { title: "Stan", id: "stan", hasGardenSize: false, olixCategory: 24},
+    { title: "Vikendica", id: "vikendica", hasGardenSize: true, olixCategory: 26 }
 ];
 
 const sizes = [

app/services/crawlerService.js

@@ -1,5 +1,6 @@
 
 
+var http = require('http');
 const crawlers = [
     //new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
   ];

package.json

@@ -7,6 +7,7 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
+    "scheduler": "node ./app/services/crawlerService.js",
     "migrate": "cd app && npx sequelize db:migrate",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",