Fixed gardenSize

app/helpers/crawlers/olxClawler.js

@@ -1,6 +1,6 @@
 const fetch = require('node-fetch');
 const cheerio = require('cheerio');
-const { allRERequest, findPointInsideBoundingBox } = require('../url');
+const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
 const { getRealEstateTypeEnum } = require('../enums');
 const { getRegion, getMunicipality } = require('../codes')
 const Promise = require("bluebird");
@@ -92,9 +92,9 @@ module.exports = class OlxCrawler {
                 // category: category,
                 url,
                 title,
-                price: isNaN(parsedPrice) ? price : parsedPrice,
+                price: isNaN(parsedPrice) ? 0 : parsedPrice,
                 size: parseFloat(size),
-                gardenSize: parseFloat(gardenSize),
+                gardenSize: isNaN(parseFloat(gardenSize)) ?  0 : parseFloat(gardenSize),
                 address,
                 region,
                 municipality,
@@ -118,6 +118,7 @@ module.exports = class OlxCrawler {
 
     async indexPage(olxUrl, maxResults = 1000) {
         try {
+            //TODO fix paging
             // console.log('Starting to index page: ' + pageNr);
             // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
 
@@ -142,7 +143,6 @@ module.exports = class OlxCrawler {
                 if (singleData) {
                     results.push(singleData);
                 }
-                // await this.sleep(500);
             }
 
             return results;
@@ -152,39 +152,37 @@ module.exports = class OlxCrawler {
     }
 
     getCategoryId (category) {
-        if (category === 'Stanovi') {
-          return 'stan';
-        } else if (category === 'Vikendice') {
-          return 'vikendica';
-        } else if (category === 'Kuće') {
-          return 'kuca';
-        }
+
+        switch(category) {
+            case 'Stanovi':
+              return 'stan';
+
+            case 'Vikendice':
+              return 'vikendica'
+
+            case 'Kuće':
+                return 'kuca';
+
+            default:
+             return '';
+          }
       }
 
     async indexPages(urls, start, end, maxResults = 1000) {
+        //TODO fix paging
         // let results = {};
         // for (let i = start; i <= end; i++) {
         //     let result = await this.indexPage(i, maxResults);
         //     Object.assign(results, result)
-        //     await this.sleep(5000);
         // }
         // return results;
 
         let results = [];
-        const indexers= [];
-        let it = 3
         for (let url of urls) {
-            // let result = await this.indexPage(url, maxResults);
-            // results.push(result);
-            it++
-            indexers.push(new Indexer(it * 2000));
-
+            let result = await this.indexPage(url, maxResults);
+            results.push(result);
         }
-        return Promise.map(indexers, function (indexer) {
-            return indexer.indexPage();
-        }).then(async (results) => {
-            return results
-        })       
+        return results;
     }
 
     async crawl() {
@@ -195,25 +193,21 @@ module.exports = class OlxCrawler {
         let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
 
         for (const result of results) {
-            for (const re1 of result) {
-                if (re1.lat !== undefined && re1.lat !== null && re1.lat !== "") {
-                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
+            for (const finalResult of result) {
+                if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat]);
 
                     if (pointInsideBoundingBox[0].length !== 0) {
-                        filteredResults.push(re1);
+                        filteredResults.push(finalResult);
                     }
                 }
             }
         }
-        // await this.sleep(10000);
+
         console.log(filteredResults);
         return filteredResults;
     }
 
-    async sleep(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
-      }
-
     createRequestUrls(realestateRequests) {
         const urls = []
 
@@ -227,7 +221,7 @@ module.exports = class OlxCrawler {
             const priceMax = "do=" + request.priceMax;
 
             const olxUrl = {
-                url: "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax,
+                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
                 email: request.email
             }
             console.log(olxUrl.url);
@@ -238,170 +232,3 @@ module.exports = class OlxCrawler {
     }
 };
 
-
-class Indexer {
-    
-    constructor(olxUrl, email, uuid) {
-        this.olxUrl = olxUrl
-        this.email = email
-        this.uuid = uuid
-    }
-
-    async indexPage() {
-
-        try {
-            // console.log('Starting to index page: ' + pageNr);
-            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
-
-            // const res = await fetch(this.olxUrl.url);
-            // const body = await res.text();
-            // const $ = cheerio.load(body);
-            // const hrefs = [];
-            // const results = [];
-        
-            // $('#rezultatipretrage').find('.listitem').each((i, elem) => {
-            //     const href = $(elem).find('a').first().attr('href');
-            //     hrefs.push(href);
-            // });
-
-            // let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
-
-            // for (let i = 0; i < hrefs.length; i++) {
-            //     console.log(`indexing: ${hrefs[i]}`);
-
-            //     const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email);
-
-            //     if (singleData) {
-            //         results.push(singleData);
-            //     }
-            //     // await this.sleep(500);
-            // }
-            await this.sleep(this.olxUrl);
-            console.log('Finished indexing PAGE');
-            const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
-            return Promise.map(singleIndex, function (indexer) {
-                return indexer.indexSingle();
-            }).then(async (results) => {
-               
-                return results
-            })  
-
-
-          
-            // return results;
-        } catch (e) {
-            console.error('Exception caught:' + e);
-        }
-    }
-
-    async indexSingle() {
-        // try {
-        //     const res = await fetch(url);
-        //     const body = await res.text();
-        //     const $ = cheerio.load(body);
-
-        //     //TODO figure out what to do with username
-        //     const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
-
-        //     // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
-        //     //     return null;
-        //     // }
-
-        //     //TODO remove properties that are not needed, and add some if they are missing
-        //     const title = $('#naslovartikla').text();
-        //     const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
-
-        //     const price = $('#pc > p:nth-child(2)').text();
-        //     const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
-        //     const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
-        //     const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
-        //     const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
-        //     const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
-
-        //     const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
-        //     const time = $('time').attr('datetime');
-        //     const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
-
-        //     const descriptions = $('.artikal_detaljniopis_tekst');
-        //     // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
-        //     const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
-        //     const imgRe = /href":("[^"]*")/g;
-        //     const matches = latLngRe.exec(body);
-        //     let lng = '',
-        //         lat = '';
-
-
-        //     const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
-        //     const parsePrice = (price) => parseFloat(price.replace(".", ""))
-
-
-        //     // TODO we dont save images ??
-
-        //     // const images = [];
-        //     // const imgMatches = body.match(imgRe);
-
-        //     // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
-        //     //     let img = imgMatches[i].replace("href\":", "")
-        //     //     img = img.replace("\"", "");
-        //     //     img = img.replace("\"", "");
-        //     //     images.push(img);
-        //     // }
-
-        //     // const uploadPromises = images.map(img => {
-        //     //     const imgFixed = eval(`'${img}'`);
-        //     //     return cloudinary.uploader.upload(eval(`'${img}'`));
-        //     // });
-
-        //     // const uploadResults = await Promise.all(uploadPromises);
-        //     // const cloudinaryImages = uploadResults.map(ur => ur.url);
-
-        //     if (matches && matches.length >= 3) {
-        //         lat = matches[1];
-        //         lng = matches[2];
-        //     }
-
-        //     const parsedPrice = parsePrice(price);
-
-        //     const locationArray = location.split(",");
-        //     const region = locationArray[0];
-        //     const municipality = locationArray[1];
-
-        //     const data = {
-        //         realEstateType: this.getCategoryId(realEstateType),
-        //         email : email,
-        //         olxId: olxId,
-        //         // category: category,
-        //         url,
-        //         title,
-        //         price: isNaN(parsedPrice) ? price : parsedPrice,
-        //         size: parseFloat(size),
-        //         gardenSize: parseFloat(gardenSize),
-        //         address,
-        //         region,
-        //         municipality,
-        //         // adType: AD_TYPE_SALE,
-        //         time,
-        //         shortDescription: descriptions.first().text(),
-        //         longDescription: descriptions.last().text(),
-        //         lat,
-        //         lng,
-        //         loc: [parseFloat(lat), parseFloat(lng)],
-        //         // images: cloudinaryImages
-        //     };
-
-        //     return data;
-        // } catch (e) {
-        //     console.error('Exception caught: ' + e.message);
-        // }
-
-        // return null;
-        await this.sleep(this.olxUrl);
-        console.log("Finished indexing single page");
-        return {};
-    }
-
-    async sleep(ms) {
-        console.log(ms);
-        return new Promise(resolve => setTimeout(resolve, ms));
-      }
-}

app/helpers/db/dbHelper.js

@@ -1,10 +1,15 @@
+const db = require('../../models/index');
 
-// const db = require('../../models/index');
+// TODO Fetch only subscribed realestate requests
+const allRERequest = async () => {
+  return await db.RealEstateRequest.findAll();
+}
 
+const findPointInsideBoundingBox = async (latLng) => {
+  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+}
 
-// const bulkInsert = async (reuslts) => {
-//     db.MarketAlert.bulkCreate({
-
-//     })
-   
-//   }
+module.exports = {
+  allRERequest,
+  findPointInsideBoundingBox
+};

app/services/crawlerService.js

@@ -4,13 +4,9 @@ const OlxCrawler = require("../helpers/crawlers/olxClawler");
 const db = require("../models/index");
 
 const olxCrawler = new OlxCrawler(1, 2, 3);
-const olxCrawler1 = new OlxCrawler(1, 2, 3);
-const olxCrawler2 = new OlxCrawler(1, 2, 3);
 
 const crawlers = [
     olxCrawler,
-    olxCrawler1,
-    olxCrawler2,
 ];
 
 async function crawlAll() {
@@ -42,6 +38,7 @@ async function crawlAll() {
                 })
             }
             try {
+                console.log(marketAlerts);
                 const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
                 await db.MarketAlert.bulkCreate(filteredMarketAlerts);
                 process.exit()