Olix crawling, filter crawling result by lat, lng

app/helpers/crawlers/olixClawler.js

@@ -1,163 +0,0 @@
-const fetch = require('node-fetch');
-const cheerio = require('cheerio');
-
-export default class OlxCrawler {
-
-  constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
-    this.fromPage = fromPage;
-    this.toPage = toPage;
-    this.maxResults = maxResults;
-  }
-
-  async indexSingle(url) {
-    try {
-      const res = await fetch(url);
-      const body = await res.text();
-      const $ = cheerio.load(body);
-
-      const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
-
-      if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
-        return null;
-      }
-
-      const title = $('#naslovartikla').text();
-      const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
-
-      const price = $('#pc > p:nth-child(2)').text();
-      const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
-      const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
-      const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
-      const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
-
-      const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
-      const time = $('time').attr('datetime');
-      const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
-
-      const descriptions = $('.artikal_detaljniopis_tekst');
-      const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
-      const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
-      const imgRe = /href":("[^"]*")/g;
-      const matches = latLngRe.exec(body);
-      let lng = '',
-        lat = '';
-
-      const images = [];
-      const imgMatches = body.match(imgRe);
-
-      const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
-      const parsePrice = (price) => parseFloat(price.replace(".", ""))
-
-
-      for (let i = 0; imgMatches && i < imgMatches.length; i++) {
-        let img = imgMatches[i].replace("href\":", "")
-        img = img.replace("\"", "");
-        img = img.replace("\"", "");
-        images.push(img);
-      }
-
-      const uploadPromises = images.map(img => {
-        const imgFixed = eval(`'${img}'`);
-        return cloudinary.uploader.upload(eval(`'${img}'`));
-      });
-
-      const uploadResults = await Promise.all(uploadPromises);
-      const cloudinaryImages = uploadResults.map(ur => ur.url);
-
-      if (matches && matches.length >= 3) {
-        lat = matches[1];
-        lng = matches[2];
-      }
-
-      const parsedPrice = parsePrice(price);
-      let parsedRooms;
-
-      if (rooms === 'Garsonjera') {
-        parsedRooms = 0;
-      } else {
-        parsedRooms = parseRooms(rooms);
-      }
-
-      const data = {
-        category: this.getCategoryId(category),
-        url,
-        title,
-        price: isNaN(parsedPrice) ? price : parsedPrice,
-        size: parseFloat(size),
-        rooms: parsedRooms,
-        floor: parseInt(floor),
-        address,
-        location,
-        adType: AD_TYPE_SALE,
-        time,
-        shortDescription: descriptions.first().text(),
-        longDescription: descriptions.last().text(),
-        lat,
-        lng,
-        loc: [parseFloat(lat), parseFloat(lng)],
-        images: cloudinaryImages
-      };
-
-      return data;
-    } catch (e) {
-      console.error('Exception caught: ' + e.message);
-    }
-
-    return null;
-  }
-
-  async indexPage(url, pageNr, maxResults = 1000) {
-    try {
-      console.log('Starting to index page: ' + pageNr);
-      const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
-
-      const res = await fetch(url);
-      const body = await res.text();
-      const $ = cheerio.load(body);
-      const hrefs = [];
-      const results = {};
-
-      $('#rezultatipretrage').find('.listitem').each((i, elem) => {
-        const href = $(elem).find('a').first().attr('href');
-        hrefs.push(href);
-      });
-
-      let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
-
-      for (let i = 0; i < hrefs.length; i++) {
-        console.log(`indexing: ${hrefs[i]}`);
-
-        const singleData = await this.indexSingle(hrefs[i]);
-
-        if (singleData) {
-          results[hrefs[i]] = singleData;
-        }
-        await this.sleep(500);
-      }
-
-      return results;
-    } catch (e) {
-      console.error('Exception caught:' + e);
-    }
-  }
-
-  async sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-
-  async indexPages(url, start, end, maxResults = 1000) {
-    let results = {};
-    for (let i = start; i <= end; i++) {
-      let result = await this.indexPage(i, maxResults);
-      Object.assign(results, result)
-      await this.sleep(5000);
-    }
-    return results;
-  }
-
-  async crawl() {
-    // TODO create URLS from db
-    let results = await this.indexPages(this.fromPage, this.toPage, this.maxResults);
-    return results;
-  }
-}

app/helpers/crawlers/olxClawler.js

@@ -0,0 +1,230 @@
+const fetch = require('node-fetch');
+const cheerio = require('cheerio');
+const { allRERequest, findPointInsideBoundingBox } = require('../url');
+const { getRealEstateTypeEnum } = require('../enums');
+const { getRegion, getMunicipality } = require('../codes')
+
+module.exports = class OlxCrawler {
+
+    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+        this.fromPage = fromPage;
+        this.toPage = toPage;
+        this.maxResults = maxResults;
+    }
+
+    async indexSingle(url) {
+        try {
+            const res = await fetch(url);
+            const body = await res.text();
+            const $ = cheerio.load(body);
+
+            const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+            // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+            //     return null;
+            // }
+
+            const title = $('#naslovartikla').text();
+            const category = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+            const price = $('#pc > p:nth-child(2)').text();
+            const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+            const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+            const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+            const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+            const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+            const time = $('time').attr('datetime');
+            const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+            const descriptions = $('.artikal_detaljniopis_tekst');
+            const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+            const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+            const imgRe = /href":("[^"]*")/g;
+            const matches = latLngRe.exec(body);
+            let lng = '',
+                lat = '';
+
+            const images = [];
+            const imgMatches = body.match(imgRe);
+
+            const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+            const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+            for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+                let img = imgMatches[i].replace("href\":", "")
+                img = img.replace("\"", "");
+                img = img.replace("\"", "");
+                images.push(img);
+            }
+
+            // const uploadPromises = images.map(img => {
+            //     const imgFixed = eval(`'${img}'`);
+            //     return cloudinary.uploader.upload(eval(`'${img}'`));
+            // });
+
+            // const uploadResults = await Promise.all(uploadPromises);
+            // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+            if (matches && matches.length >= 3) {
+                lat = matches[1];
+                lng = matches[2];
+            }
+
+            const parsedPrice = parsePrice(price);
+            let parsedRooms;
+
+            if (rooms === 'Garsonjera') {
+                parsedRooms = 0;
+            } else {
+                parsedRooms = parseRooms(rooms);
+            }
+
+            const data = {
+                // category: this.getCategoryId(category),
+                category: category,
+                url,
+                title,
+                price: isNaN(parsedPrice) ? price : parsedPrice,
+                size: parseFloat(size),
+                rooms: parsedRooms,
+                floor: parseInt(floor),
+                address,
+                location,
+                // adType: AD_TYPE_SALE,
+                time,
+                shortDescription: descriptions.first().text(),
+                longDescription: descriptions.last().text(),
+                lat,
+                lng,
+                loc: [parseFloat(lat), parseFloat(lng)],
+                // images: cloudinaryImages
+            };
+
+            return data;
+        } catch (e) {
+            console.error('Exception caught: ' + e.message);
+        }
+
+        return null;
+    }
+
+
+
+    async indexPage(url, maxResults = 1000) {
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+            const res = await fetch(url);
+            const body = await res.text();
+            const $ = cheerio.load(body);
+            const hrefs = [];
+            const results = [];
+
+            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+                const href = $(elem).find('a').first().attr('href');
+                hrefs.push(href);
+            });
+
+            let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+            for (let i = 0; i < hrefs.length; i++) {
+                console.log(`indexing: ${hrefs[i]}`);
+
+                const singleData = await this.indexSingle(hrefs[i]);
+
+                if (singleData) {
+                    results.push(singleData);
+                }
+                await this.sleep(500);
+            }
+
+            return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
+        }
+    }
+
+    // getCategoryId (category) {
+    //     if (category === 'Stanovi') {
+    //       return CATEGORY_FLAT;
+    //     } else if (category === 'Zemljišta') {
+    //       return CATEGORY_LAND;
+    //     } else if (category === 'Kuće') {
+    //       return CATEGORY_HOUSE;
+    //     } else if (category === 'Poslovni prostori') {
+    //       return CATEGORY_OFFICE;
+    //     }
+    //   }
+
+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+
+    async indexPages(urls, start, end, maxResults = 1000) {
+        // let results = {};
+        // for (let i = start; i <= end; i++) {
+        //     let result = await this.indexPage(i, maxResults);
+        //     Object.assign(results, result)
+        //     await this.sleep(5000);
+        // }
+        // return results;
+
+        let results = [];
+        for (let url of urls) {
+            let result = await this.indexPage(url, maxResults);
+            // Object.assign(results, result)
+            results.push(result);
+            await this.sleep(5000);
+        }
+        return results;
+    }
+
+    async crawl() {
+        // TODO create URLS from db
+        const filteredResults = [];
+        const realestateRequests = await allRERequest()
+        const urls = this.createRequestUrls(realestateRequests);
+        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
+
+        for (const result of results) {
+            console.log(result);
+            for (const re1 of result) {
+                if (re1.lat !== undefined) {
+                    console.log(re1.lat);
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
+                    console.log(pointInsideBoundingBox);
+                }
+            }
+        }
+        // console.log(results);
+
+
+
+        return results;
+    }
+
+    createRequestUrls(realestateRequests) {
+        const urls = []
+
+        for (const request of realestateRequests) {
+            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
+            const region = "kanton=" + getRegion(request.region).olxid;
+            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
+            const sizeMin = "kvadrata_min=" + request.sizeMin;
+            const sizeMax = "kvadrata_max=" + request.sizeMax;
+            const priceMin = "od=" + request.priceMin;
+            const priceMax = "do=" + request.priceMax;
+
+
+
+            const olxUrl = "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax
+            console.log(olxUrl);
+            urls.push(olxUrl);
+        }
+
+        return urls;
+    }
+};

app/helpers/enums.js

@@ -1,7 +1,7 @@
 const realEstateTypes = [
-    { title: "Kuća", id: "kuca", hasGardenSize: true, olixCategory: 23 },
+    { title: "Kuća", id: "kuca", hasGardenSize: true, olxCategory: 24 },
-    { title: "Stan", id: "stan", hasGardenSize: false, olixCategory: 24},
+    { title: "Stan", id: "stan", hasGardenSize: false, olxCategory: 23},
-    { title: "Vikendica", id: "vikendica", hasGardenSize: true, olixCategory: 26 }
+    { title: "Vikendica", id: "vikendica", hasGardenSize: true, olxCategory: 26 }
 ];
 
 const sizes = [

app/helpers/url.js

@@ -8,6 +8,16 @@ const currentRERequest = async (req) => {
   return request;
 };
 
+const allRERequest = async () => {
+  return await db.RealEstateRequest.findAll();
+}
+
+const findPointInsideBoundingBox = async (latLng) => {
+  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+}
+
 module.exports = {
-  currentRERequest
+  currentRERequest,
+  allRERequest,
+  findPointInsideBoundingBox
 };

app/services/crawlerService.js

@@ -1,8 +1,10 @@
 
+// import OlxCrawler from '../helpers/crawlers/olixClawler'
+const OlxCrawler = require("../helpers/crawlers/olxClawler");
 
-var http = require('http');
 const crawlers = [
-    //new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
+    new OlxCrawler(1, 2, 3),
+    // new OlxCrawler(process.env.OLX_FROM_PAGE, process.env.OLX_TO_PAGE, process.env.OLX_MAX_RESULTS),
   ];
 
   async function crawlAll() {