experiment

Range slider

See merge request saburly/marketalarm/web!16

.gitignore

@@ -1 +1,2 @@
 node_modules/
+.env

app/controllers/gardenSizes.js

@@ -2,6 +2,8 @@ const { currentRERequest } = require('../helpers/url');
 const { getRealEstateTypeEnum } = require('../helpers/enums');
 
 const getGardenSize = (req,res) => {
+
+  const unit = " m2"
   const rangeFrom = {
     min : 10,
     max : 3000,
@@ -15,7 +17,8 @@ const getGardenSize = (req,res) => {
     value : 100,
     step : 10
   }
-  res.render('gardenSize', { rangeFrom, rangeTo });
+
+  res.render('gardenSize', { rangeFrom, rangeTo, unit });
 };
 
 const postGardenSize = async (req, res) => {

app/controllers/prices.js

@@ -1,6 +1,8 @@
 const { currentRERequest } = require('../helpers/url');
 
 const getPrice = (req,res) => {
+
+  const unit = " KM"
   const rangeFrom = {
     min : 1000,
     max : 250000,
@@ -16,7 +18,7 @@ const getPrice = (req,res) => {
   }
 
 
-  res.render('price', {rangeFrom, rangeTo  });
+  res.render('price', {rangeFrom, rangeTo, unit  });
 };
 
 const postPrice = async (req, res) => {

app/controllers/querySubmit.js

@@ -1,5 +1,6 @@
 const { currentRERequest } = require('../helpers/url');
 const { isValidEmail } = require('../helpers/email');
+const { sendTemplatedEmail} = require('../helpers/awsEmail');
 
 const getQuerySubmit = async (req, res) => {
   const nextStep = req.query.nextStep;
@@ -38,7 +39,9 @@ const postQuerySubmit = async (req, res) => {
   }
 
   request.email = req.body.email;
+  request.subscribed = true;
   await request.save();
+  sendTemplatedEmail(req.body.email, request);
   res.redirect(nextStep); 
 };
 

app/controllers/sizes.js

@@ -2,6 +2,8 @@ const { currentRERequest } = require('../helpers/url');
 const { sizes, getRealEstateTypeEnum } = require('../helpers/enums');
 
 const getSize = (req,res) => {
+
+  const unit = " m2"
   const rangeFrom = {
     min : 10,
     max : 250,
@@ -16,7 +18,7 @@ const getSize = (req,res) => {
     step : 10
   }
 
-  res.render('size', { rangeFrom, rangeTo });
+  res.render('size', { rangeFrom, rangeTo, unit });
 };
 
 const postSize = async (req, res) => {

app/controllers/unsubscribe.js

@@ -0,0 +1,15 @@
+
+const { currentRERequest } = require('../helpers/url');
+
+const getUnsubscribe = async (req, res) => {
+    const request = await currentRERequest(req);
+    request.subscribed = false;
+    await request.save();
+
+    res.render('unsubscribe', { nextStep: '/vrstanekretnine' });
+};
+
+
+module.exports = {
+    getUnsubscribe
+};

app/helpers/awsEmail.js

@@ -0,0 +1,96 @@
+
+const dotenv = require('dotenv');
+dotenv.config();
+const { getRealEstateTypeEnum } = require('./enums');
+const { getRegionName, getMunicipalityName } = require('./codes');
+const AWS = require('aws-sdk');
+AWS.config.update({
+  region: process.env.AMAZON_REGION,
+  credentials:
+  {
+    accessKeyId: process.env.AMAZON_ACCES_KEY_ID,
+    secretAccessKey: process.env.AMAZON_SECRET_ACCESS_KEY
+  }
+});
+
+const sendTemplatedEmail = async (email, request) => {
+
+  const params = {
+    Destination: { /* required */
+      CcAddresses: [
+      ],
+      ToAddresses: [
+        email
+      ]
+    },
+    Message: { /* required */
+      Body: { /* required */
+        Html: {
+          Charset: "UTF-8",
+          Data: getEmailHTML(request)
+        },
+        Text: {
+          Charset: "UTF-8",
+          Data: getEmaiTextVersion(request)
+        }
+      },
+      Subject: {
+        Charset: 'UTF-8',
+        Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}`
+      }
+    },
+    Source: process.env.SOURCE_EMAIL, /* required */
+    ReplyToAddresses: [
+      process.env.SOURCE_EMAIL,
+    ],
+  };
+
+  const sendEmailPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendEmail(params).promise();
+  await sendEmailPromise;
+}
+
+const getEmailHTML = (realestateRequest) => {
+  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
+  const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``
+
+  return `<h1> Zdravo, 
+  Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima. </h1>
+  <h2> Ovo je tražena nekretnina:  </h2> 
+  <div>
+  <div> <strong>Tip nekretnine: ${realEstateType.title} </strong></div>
+  <div><strong>Područje: ${getRegionName(realestateRequest.region)} </strong></div>
+  <div><strong>Mjesto: ${getMunicipalityName(realestateRequest.region, realestateRequest.municipality)} </strong></div>
+  <div><strong>Kvadratura nekretnine: Od ${realestateRequest.sizeMin} do ${realestateRequest.sizeMax} m2  </strong></div>
+  ${gardenSize}
+  <div><strong>Cijena: ${realestateRequest.priceMin} do ${realestateRequest.priceMax} KM  </strong></div>
+  </div>
+  <div> 
+
+  </div>
+  <div><strong> Ako želis prestati dobijati obavještenja za ovu pretragu klikni ${process.env.APP_URL}/odjava/${realestateRequest.uniqueId} </strong></div>
+  <div><strong>Ako želiš promijeniti uslove pretrage klikni  ${process.env.APP_URL}/pregled/${realestateRequest.uniqueId} </strong></div>
+<h4> Tvoj,
+Javimi tim.  
+</h4>`
+
+}
+
+const getEmaiTextVersion = (realestateRequest) => {
+  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
+  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax  : ""
+
+  const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
+    + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
+    + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
+    + gardenSize
+    "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
+    "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
+    "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
+    "\n Tvoj,\n Javimi tim"
+
+  return text;
+}
+
+module.exports = {
+  sendTemplatedEmail
+};

app/helpers/crawlers/olxClawler.js

@@ -0,0 +1,407 @@
+const fetch = require('node-fetch');
+const cheerio = require('cheerio');
+const { allRERequest, findPointInsideBoundingBox } = require('../url');
+const { getRealEstateTypeEnum } = require('../enums');
+const { getRegion, getMunicipality } = require('../codes')
+const Promise = require("bluebird");
+
+module.exports = class OlxCrawler {
+    //TODO figure best way to handle paging
+    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
+        this.fromPage = fromPage;
+        this.toPage = toPage;
+        this.maxResults = maxResults;
+    }
+
+    async indexSingle(url, email) {
+        try {
+            const res = await fetch(url);
+            const body = await res.text();
+            const $ = cheerio.load(body);
+
+            //TODO figure out what to do with username
+            const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+            // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+            //     return null;
+            // }
+
+            //TODO remove properties that are not needed, and add some if they are missing
+            const title = $('#naslovartikla').text();
+            const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+            const price = $('#pc > p:nth-child(2)').text();
+            const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+            const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+            const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+            const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
+            const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+            const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+            const time = $('time').attr('datetime');
+            const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+            const descriptions = $('.artikal_detaljniopis_tekst');
+            // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+            const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+            const imgRe = /href":("[^"]*")/g;
+            const matches = latLngRe.exec(body);
+            let lng = '',
+                lat = '';
+
+
+            const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+            const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+            // TODO we dont save images ??
+
+            // const images = [];
+            // const imgMatches = body.match(imgRe);
+
+            // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+            //     let img = imgMatches[i].replace("href\":", "")
+            //     img = img.replace("\"", "");
+            //     img = img.replace("\"", "");
+            //     images.push(img);
+            // }
+
+            // const uploadPromises = images.map(img => {
+            //     const imgFixed = eval(`'${img}'`);
+            //     return cloudinary.uploader.upload(eval(`'${img}'`));
+            // });
+
+            // const uploadResults = await Promise.all(uploadPromises);
+            // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+            if (matches && matches.length >= 3) {
+                lat = matches[1];
+                lng = matches[2];
+            }
+
+            const parsedPrice = parsePrice(price);
+
+            const locationArray = location.split(",");
+            const region = locationArray[0];
+            const municipality = locationArray[1];
+
+            const data = {
+                realEstateType: this.getCategoryId(realEstateType),
+                email : email,
+                olxId: olxId,
+                // category: category,
+                url,
+                title,
+                price: isNaN(parsedPrice) ? price : parsedPrice,
+                size: parseFloat(size),
+                gardenSize: parseFloat(gardenSize),
+                address,
+                region,
+                municipality,
+                // adType: AD_TYPE_SALE,
+                time,
+                shortDescription: descriptions.first().text(),
+                longDescription: descriptions.last().text(),
+                lat,
+                lng,
+                loc: [parseFloat(lat), parseFloat(lng)],
+                // images: cloudinaryImages
+            };
+
+            return data;
+        } catch (e) {
+            console.error('Exception caught: ' + e.message);
+        }
+
+        return null;
+    }
+
+    async indexPage(olxUrl, maxResults = 1000) {
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+            const res = await fetch(olxUrl.url);
+            const body = await res.text();
+            const $ = cheerio.load(body);
+            const hrefs = [];
+            const results = [];
+
+            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+                const href = $(elem).find('a').first().attr('href');
+                hrefs.push(href);
+            });
+
+            let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+            for (let i = 0; i < hrefs.length; i++) {
+                console.log(`indexing: ${hrefs[i]}`);
+
+                const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
+
+                if (singleData) {
+                    results.push(singleData);
+                }
+                // await this.sleep(500);
+            }
+
+            return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
+        }
+    }
+
+    getCategoryId (category) {
+        if (category === 'Stanovi') {
+          return 'stan';
+        } else if (category === 'Vikendice') {
+          return 'vikendica';
+        } else if (category === 'Kuće') {
+          return 'kuca';
+        }
+      }
+
+    async indexPages(urls, start, end, maxResults = 1000) {
+        // let results = {};
+        // for (let i = start; i <= end; i++) {
+        //     let result = await this.indexPage(i, maxResults);
+        //     Object.assign(results, result)
+        //     await this.sleep(5000);
+        // }
+        // return results;
+
+        let results = [];
+        const indexers= [];
+        let it = 3
+        for (let url of urls) {
+            // let result = await this.indexPage(url, maxResults);
+            // results.push(result);
+            it++
+            indexers.push(new Indexer(it * 2000));
+
+        }
+        return Promise.map(indexers, function (indexer) {
+            return indexer.indexPage();
+        }).then(async (results) => {
+            return results
+        })       
+    }
+
+    async crawl() {
+
+        const filteredResults = [];
+        const realestateRequests = await allRERequest()
+        const urls = this.createRequestUrls(realestateRequests);
+        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
+
+        for (const result of results) {
+            for (const re1 of result) {
+                if (re1.lat !== undefined && re1.lat !== null && re1.lat !== "") {
+                    const pointInsideBoundingBox = await findPointInsideBoundingBox([re1.lng, re1.lat]);
+
+                    if (pointInsideBoundingBox[0].length !== 0) {
+                        filteredResults.push(re1);
+                    }
+                }
+            }
+        }
+        // await this.sleep(10000);
+        console.log(filteredResults);
+        return filteredResults;
+    }
+
+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+
+    createRequestUrls(realestateRequests) {
+        const urls = []
+
+        for (const request of realestateRequests) {
+            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
+            const region = "kanton=" + getRegion(request.region).olxid;
+            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
+            const sizeMin = "kvadrata_min=" + request.sizeMin;
+            const sizeMax = "kvadrata_max=" + request.sizeMax;
+            const priceMin = "od=" + request.priceMin;
+            const priceMax = "do=" + request.priceMax;
+
+            const olxUrl = {
+                url: "https://www.olx.ba/pretraga?" + realsestateType + "&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&" + region + "&" + municipality + "&" + priceMin + "&" + priceMax + "&vrsta=samoprodaja&" + sizeMin + "&" + sizeMax,
+                email: request.email
+            }
+            console.log(olxUrl.url);
+            urls.push(olxUrl);
+        }
+
+        return urls;
+    }
+};
+
+
+class Indexer {
+    
+    constructor(olxUrl, email, uuid) {
+        this.olxUrl = olxUrl
+        this.email = email
+        this.uuid = uuid
+    }
+
+    async indexPage() {
+
+        try {
+            // console.log('Starting to index page: ' + pageNr);
+            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
+
+            // const res = await fetch(this.olxUrl.url);
+            // const body = await res.text();
+            // const $ = cheerio.load(body);
+            // const hrefs = [];
+            // const results = [];
+        
+            // $('#rezultatipretrage').find('.listitem').each((i, elem) => {
+            //     const href = $(elem).find('a').first().attr('href');
+            //     hrefs.push(href);
+            // });
+
+            // let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
+
+            // for (let i = 0; i < hrefs.length; i++) {
+            //     console.log(`indexing: ${hrefs[i]}`);
+
+            //     const singleData = await this.indexSingle(hrefs[i], this.olxUrl.email);
+
+            //     if (singleData) {
+            //         results.push(singleData);
+            //     }
+            //     // await this.sleep(500);
+            // }
+            await this.sleep(this.olxUrl);
+            console.log('Finished indexing PAGE');
+            const singleIndex = [new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl), new Indexer(this.olxUrl)]
+            return Promise.map(singleIndex, function (indexer) {
+                return indexer.indexSingle();
+            }).then(async (results) => {
+               
+                return results
+            })  
+
+
+          
+            // return results;
+        } catch (e) {
+            console.error('Exception caught:' + e);
+        }
+    }
+
+    async indexSingle() {
+        // try {
+        //     const res = await fetch(url);
+        //     const body = await res.text();
+        //     const $ = cheerio.load(body);
+
+        //     //TODO figure out what to do with username
+        //     const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
+
+        //     // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
+        //     //     return null;
+        //     // }
+
+        //     //TODO remove properties that are not needed, and add some if they are missing
+        //     const title = $('#naslovartikla').text();
+        //     const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
+
+        //     const price = $('#pc > p:nth-child(2)').text();
+        //     const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
+        //     const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
+        //     const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
+        //     const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
+        //     const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
+
+        //     const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
+        //     const time = $('time').attr('datetime');
+        //     const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
+
+        //     const descriptions = $('.artikal_detaljniopis_tekst');
+        //     // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
+        //     const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+        //     const imgRe = /href":("[^"]*")/g;
+        //     const matches = latLngRe.exec(body);
+        //     let lng = '',
+        //         lat = '';
+
+
+        //     const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
+        //     const parsePrice = (price) => parseFloat(price.replace(".", ""))
+
+
+        //     // TODO we dont save images ??
+
+        //     // const images = [];
+        //     // const imgMatches = body.match(imgRe);
+
+        //     // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
+        //     //     let img = imgMatches[i].replace("href\":", "")
+        //     //     img = img.replace("\"", "");
+        //     //     img = img.replace("\"", "");
+        //     //     images.push(img);
+        //     // }
+
+        //     // const uploadPromises = images.map(img => {
+        //     //     const imgFixed = eval(`'${img}'`);
+        //     //     return cloudinary.uploader.upload(eval(`'${img}'`));
+        //     // });
+
+        //     // const uploadResults = await Promise.all(uploadPromises);
+        //     // const cloudinaryImages = uploadResults.map(ur => ur.url);
+
+        //     if (matches && matches.length >= 3) {
+        //         lat = matches[1];
+        //         lng = matches[2];
+        //     }
+
+        //     const parsedPrice = parsePrice(price);
+
+        //     const locationArray = location.split(",");
+        //     const region = locationArray[0];
+        //     const municipality = locationArray[1];
+
+        //     const data = {
+        //         realEstateType: this.getCategoryId(realEstateType),
+        //         email : email,
+        //         olxId: olxId,
+        //         // category: category,
+        //         url,
+        //         title,
+        //         price: isNaN(parsedPrice) ? price : parsedPrice,
+        //         size: parseFloat(size),
+        //         gardenSize: parseFloat(gardenSize),
+        //         address,
+        //         region,
+        //         municipality,
+        //         // adType: AD_TYPE_SALE,
+        //         time,
+        //         shortDescription: descriptions.first().text(),
+        //         longDescription: descriptions.last().text(),
+        //         lat,
+        //         lng,
+        //         loc: [parseFloat(lat), parseFloat(lng)],
+        //         // images: cloudinaryImages
+        //     };
+
+        //     return data;
+        // } catch (e) {
+        //     console.error('Exception caught: ' + e.message);
+        // }
+
+        // return null;
+        await this.sleep(this.olxUrl);
+        console.log("Finished indexing single page");
+        return {};
+    }
+
+    async sleep(ms) {
+        console.log(ms);
+        return new Promise(resolve => setTimeout(resolve, ms));
+      }
+}

app/helpers/db/dbHelper.js

@@ -0,0 +1,10 @@
+
+// const db = require('../../models/index');
+
+
+// const bulkInsert = async (reuslts) => {
+//     db.MarketAlert.bulkCreate({
+
+//     })
+   
+//   }

app/helpers/enums.js

@@ -1,7 +1,7 @@
 const realEstateTypes = [
-    { title: "Kuća", id: "kuca", hasGardenSize: true },
-    { title: "Stan", id: "stan", hasGardenSize: false },
-    { title: "Vikendica", id: "vikendica", hasGardenSize: true }
+    { title: "Kuća", id: "kuca", hasGardenSize: true, olxCategory: 24 },
+    { title: "Stan", id: "stan", hasGardenSize: false, olxCategory: 23},
+    { title: "Vikendica", id: "vikendica", hasGardenSize: true, olxCategory: 26 }
 ];
 
 const sizes = [

app/helpers/url.js

@@ -7,7 +7,17 @@ const currentRERequest = async (req) => {
   const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
   return request;
 };
+// TODO Fetch only subscribed realestate requests
+const allRERequest = async () => {
+  return await db.RealEstateRequest.findAll();
+}
+
+const findPointInsideBoundingBox = async (latLng) => {
+  return await db.sequelize.query("SELECT * FROM \"RealEstateRequests\" WHERE ST_Contains(\"RealEstateRequests\".bounding_box, ST_GEOMFROMTEXT(\'POINT (" + latLng[0] + " " + latLng[1]+ ")\'))");
+}
 
 module.exports = {
-  currentRERequest
+  currentRERequest,
+  allRERequest,
+  findPointInsideBoundingBox
 };

app/lib/sendNotification.js

@@ -1,7 +1,7 @@
 const scrapTheItems = require("./scrapTheItems");
 const convertToDate = require("./convertToDate");
 const AWS = require('aws-sdk');
-AWS.config.update({region: 'eu-central-1'});
+// AWS.config.update({region: 'eu-central-1'});
 
 
 async function sendNotification(marketAlert) {

app/migrations/20190531111232-subscribed-boolean.js

@@ -0,0 +1,18 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+      'RealEstateRequests',
+      'subscribed',
+     Sequelize.BOOLEAN
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+      'RealEstateRequests',
+      'subscribed'
+    );
+  }
+};

app/migrations/20190618103020-expand-maketalert.js

@@ -0,0 +1,37 @@
+'use strict';
+
+module.exports = {
+    up: (queryInterface, Sequelize) => {
+        return queryInterface.sequelize.transaction((t) => {
+            return Promise.all([
+                queryInterface.addColumn('MarketAlerts', 'size', {
+                    type: Sequelize.INTEGER,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'gardenSize', {
+                    type: Sequelize.INTEGER,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'price', {
+                    type: Sequelize.INTEGER,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'municipality', {
+                    type: Sequelize.STRING,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'region', {
+                    type: Sequelize.STRING,
+                }, { transaction: t })
+            ])
+        })
+    },
+
+    down: (queryInterface, Sequelize) => {
+        return queryInterface.sequelize.transaction((t) => {
+            return Promise.all([
+                queryInterface.removeColumn('MarketAlerts', 'size', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'gardenSize', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'price', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'municipality', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'region', { transaction: t })
+            ])
+        })
+    }
+};

app/migrations/20190618124522-marketalerts-additional-info.js

@@ -0,0 +1,33 @@
+'use strict';
+
+module.exports = {
+    up: (queryInterface, Sequelize) => {
+        return queryInterface.sequelize.transaction((t) => {
+            return Promise.all([
+                queryInterface.removeColumn('MarketAlerts', 'olxUrl', { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'url', {
+                    type: Sequelize.STRING,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'realestateOrigin', {
+                    type: Sequelize.STRING,
+                }, { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'originId', {
+                    type: Sequelize.STRING,
+                }, { transaction: t })
+            ])
+        })
+    },
+
+    down: (queryInterface, Sequelize) => {
+        return queryInterface.sequelize.transaction((t) => {
+            return Promise.all([
+                queryInterface.removeColumn('MarketAlerts', 'url', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'realestateOrigin', { transaction: t }),
+                queryInterface.removeColumn('MarketAlerts', 'originId', { transaction: t }),
+                queryInterface.addColumn('MarketAlerts', 'olxUrl', {
+                    type: Sequelize.STRING
+                }, { transaction: t })
+            ])
+        })
+    }
+};

app/migrations/20190621162321-add-category-to-marketalert.js

@@ -0,0 +1,20 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+        'MarketAlerts',
+        'realEstateType',
+        {
+          type: Sequelize.STRING
+        }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn(
+        'MarketAlerts',
+        'realEstateType'
+    );
+  }
+};

app/models/marketalert.js

@@ -1,8 +1,17 @@
 'use strict';
 module.exports = (sequelize, DataTypes) => {
   const MarketAlert = sequelize.define('MarketAlert', {
-    olxUrl: DataTypes.STRING,
+    url: DataTypes.STRING,
+    realestateOrigin: DataTypes.STRING,
+    originId: DataTypes.STRING,
     lastDate: DataTypes.STRING,
+    size : DataTypes.INTEGER,
+    gardenSize : DataTypes.INTEGER,
+    price : DataTypes.INTEGER,
+    municipality : DataTypes.STRING,
+    region : DataTypes.STRING,
+    realEstateType : DataTypes.STRING,
+    
     email: { 
       type: DataTypes.STRING,
       allowNul: false

app/models/realestaterequest.js

@@ -18,7 +18,8 @@ module.exports = (sequelize, DataTypes) => {
     gardenSizeMax: DataTypes.INTEGER,
     priceMin: DataTypes.INTEGER,
     priceMax: DataTypes.INTEGER,
-    bounding_box: DataTypes.GEOMETRY('POINT', 4326)
+    bounding_box: DataTypes.GEOMETRY('POINT', 4326),
+    subscribed: DataTypes.BOOLEAN
   }, {});
   RealEstateRequest.associate = function(models) {
     // associations can be defined here

app/public/main.css

@@ -9,6 +9,9 @@
   background-repeat: no-repeat;
   color: rgba(0, 0, 0, 0);
 }
+.no-ui-slider {
+  width: 95%
+}
 
 
 #map {

app/services/crawlerService.js

@@ -0,0 +1,58 @@
+
+const Promise = require("bluebird");
+const OlxCrawler = require("../helpers/crawlers/olxClawler");
+const db = require("../models/index");
+
+const olxCrawler = new OlxCrawler(1, 2, 3);
+const olxCrawler1 = new OlxCrawler(1, 2, 3);
+const olxCrawler2 = new OlxCrawler(1, 2, 3);
+
+const crawlers = [
+    olxCrawler,
+    olxCrawler1,
+    olxCrawler2,
+];
+
+async function crawlAll() {
+
+    Promise.map(crawlers, function (crawler) {
+        return crawler.crawl();
+    }).then(async (results) => {
+
+        try {
+
+            const marketAlertsFromDb = await db.MarketAlert.findAll();
+
+            const marketAlerts = [];
+            const mergedResults = [].concat.apply([], results);
+
+            for (const result of mergedResults) {
+                marketAlerts.push({
+                    url: result.url,
+                    realestateOrigin: "OLX",
+                    originId: 1,
+                    size: result.size,
+                    price: result.price,
+                    email: result.email,
+                    // lastDate: DataTypes.STRING,
+                    municipality: result.municipality,
+                    region: result.region,
+                    gardenSize: isNaN(result.gardenSize) ? 0 : result.gardenSize,
+                    realEstateType: result.realEstateType
+                })
+            }
+            try {
+                const filteredMarketAlerts =  marketAlerts.filter((elem) => !marketAlertsFromDb.find(({ url }) => elem.url === url));
+                await db.MarketAlert.bulkCreate(filteredMarketAlerts);
+                process.exit()
+            } catch (e) {
+                console.log("Could not bulkCreate marketalers reason: ", e);
+            }
+        } catch (e) {
+            console.log("Error crawling. Trying next crawler! ", e);
+        }
+    })
+};
+
+crawlAll();
+

app/views/layout.ejs

@@ -3,6 +3,7 @@
   <head>
      <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
     <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
       <script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
     <meta charset="UTF-8" />
@@ -13,5 +14,9 @@
       <%-body%>
     </div>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/js/materialize.min.js"></script> 
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.js"></script> 
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/wnumb/1.1.0/wNumb.min.js"></script> 
+    
+    
   </body>
 </html>

app/views/neighborhoodMap.ejs

@@ -2,89 +2,128 @@
   <h2>U kojem naselju tražite nekretninu?</h2>
 </div>
 
-<div class="row center-align" >
+<div class="row center-align">
   <div id="floating-panel">
     <input id="address" type="textbox" value="">
     <input id="submit" type="button" value="Trazi">
   </div>
   <div id="map"></div>
 </div>
-  <form method="POST" id="form-map-output">
-    <div class="row center-align">
-      <div class="col s6 push-s3">
-        <a id="btnsubmit" href="#" class="welcome-center-button waves-effect waves-light btn">
-          Dalje
-        </a>
-      </div>
+<form method="POST" id="form-map-output">
+  <div class="row center-align">
+    <div class="col s6 push-s3">
+      <a id="btnsubmit" href="#" class="welcome-center-button waves-effect waves-light btn">
+        Dalje
+      </a>
     </div>
-    <input type="hidden" name="north" id=north />
-    <input type="hidden" name="south" id=south />
-    <input type="hidden" name="east" id=east />
-    <input type="hidden" name="west" id=west />
-  </form>
-  <script>
+  </div>
+  <input type="hidden" name="north" id=north />
+  <input type="hidden" name="south" id=south />
+  <input type="hidden" name="east" id=east />
+  <input type="hidden" name="west" id=west />
+</form>
+<script>
 
 
-    var map;
-    var municipality = "<%= municipality%>";
-    var defaultAddress = document.getElementById('address');
+  var map;
+  var municipality = "<%= municipality%>";
+  var defaultAddress = document.getElementById('address');
+  var latLngRestrictions = [];
 
-    var BOSNIA_BOUNDS = {
-      north: 45.70,
-      south: 41.69,
-      west: 15.55,
-      east: 20.77,
-    };
+  var BOSNIA_BOUNDS = {
+    north: 45.70,
+    south: 41.69,
+    west: 15.55,
+    east: 20.77,
+  };
 
-    function initMap() {
-      map = new google.maps.Map(document.getElementById('map'), {
-        center: { lat: -34.397, lng: 150.644 },
-        zoom: 11,
-        restriction: {
-          latLngBounds: BOSNIA_BOUNDS,
-          strictBounds: false,
-        },
+  function initMap() {
+    var geocoder = new google.maps.Geocoder();
+
+    document.getElementById('submit').addEventListener('click', function () {
+      geocodeAddress(geocoder, map, false, latLngRestrictions);
+    });
+
+
+
+    defaultAddress.value = municipality;
+    geocodeAddress(geocoder, map, true);
+
+    $(document).ready(() => {
+      $("#btnsubmit").click(() => {
+        var bounds = map.getBounds();
+
+        $("#north").val(map.getBounds().getNorthEast().lat());
+        $("#south").val(map.getBounds().getSouthWest().lat());
+        $("#east").val(map.getBounds().getNorthEast().lng());
+        $("#west").val(map.getBounds().getSouthWest().lng());
+
+        $("#form-map-output").submit();
       });
-      var geocoder = new google.maps.Geocoder();
+    });
 
-      document.getElementById('submit').addEventListener('click', function () {
-        geocodeAddress(geocoder, map);
-      });
 
-      function geocodeAddress(geocoder, resultsMap) {
-        var address = document.getElementById('address').value;
-        geocoder.geocode({ 'address': address }, function (results, status) {
-          if (status === 'OK') {
-            resultsMap.setCenter(results[0].geometry.location);
-            var marker = new google.maps.Marker({
-              map: resultsMap,
-              position: results[0].geometry.location
-            });
-          } else {
-            alert('Geocode was not successful for the following reason: ' + status);
+  }
+
+  function geocodeAddress(geocoder, resultsMap, isInit, geocoderRestrictions) {
+
+
+
+    var address = document.getElementById('address').value;
+    let geocoderOptions = geocoderRestrictions
+      ? { 'address': address, 'bounds': geocoderRestrictions }
+      : { 'address': address }
+
+    geocoder.geocode(geocoderOptions, function (results, status) {
+      if (status === 'OK') {
+
+        var bounds = results[0].geometry.bounds;
+
+        var resultBounds = new google.maps.LatLngBounds(
+
+          results[0].geometry.viewport.getSouthWest(),
+          results[0].geometry.viewport.getNorthEast()
+        );
+
+        if (isInit) {
+          map = new google.maps.Map(document.getElementById('map'), {
+            zoom: 11,
+          });
+          resultsMap = map
+        }
+
+        // map.fitBounds(resultBounds);
+        resultsMap.setCenter(results[0].geometry.location);
+
+        if (isInit) {
+
+          latLngRestrictions = new google.maps.LatLngBounds(
+            new google.maps.LatLng(bounds.getSouthWest().lat(), bounds.getSouthWest().lng()),
+            new google.maps.LatLng(bounds.getNorthEast().lng(), bounds.getNorthEast().lng()));
+
+
+          let latLngRestrictionsa = {
+            west: bounds.getSouthWest().lng(),
+            east: bounds.getNorthEast().lng(),
+            north: bounds.getNorthEast().lat(),
+            south: bounds.getSouthWest().lat()
           }
-        });
+          map.setOptions({
+            restriction: {
+              latLngBounds: latLngRestrictionsa,
+              strictBounds: false,
+            }
+          })
+        } else {
+          resultsMap.setZoom(17);
+        }
+
+      } else {
+        alert('Geocode was not successful for the following reason: ' + status);
       }
-
-      defaultAddress.value = municipality;
-      geocodeAddress(geocoder, map);
-
-      $(document).ready(() => {
-        $("#btnsubmit").click(() => {
-          var bounds = map.getBounds();
-
-          $("#north").val(map.getBounds().getNorthEast().lat());
-          $("#south").val(map.getBounds().getSouthWest().lat());
-          $("#east").val(map.getBounds().getNorthEast().lng());
-          $("#west").val(map.getBounds().getSouthWest().lng());
-
-          $("#form-map-output").submit();
-        });
-      });
-
-
-    }
-  </script>
-  <script src="https://maps.googleapis.com/maps/api/js?key=AIzaSyAna8ohfV2HBMcxGk_29vqxU5Z_bDickqg&callback=initMap"
-    async defer></script>
+    });
+  }
+</script>
+<script src="https://maps.googleapis.com/maps/api/js?key=AIzaSyAna8ohfV2HBMcxGk_29vqxU5Z_bDickqg&callback=initMap" async
+  defer></script>
 </div>

app/views/partials/range.ejs

@@ -1,56 +1,53 @@
 <form method="POST" id="form-range">
 
-    <div>
-      <div class="row center-align">
-        <h6>Od</h6>
-      </div>
-      <p class="range-field">
-        <input 
-        name="from" 
-        id="from" 
-        type="range"
-         value="<%= rangeFrom.value %>" 
-         min="<%= rangeFrom.min %>"
-         max="<%= rangeFrom.max %>"  
-         step="<%= rangeFrom.step %>"/>
-      </p>
-      <div class="row center-align">
-        <h6>Do</h6>
-      </div>
-      <p class="range-field">
-        <input 
-        name="to" 
-        id="to" 
-        type="range" 
-        value="<%= rangeTo.value %>"  
-        min="<%= rangeTo.min %>" 
-        max="<%= rangeTo.max %>" 
-        step="<%= rangeTo.step %>"/>
-      </p>
-    </div>
-    <div class="col s6 push-s3">
-      <a id="btnsubmit" href="#" class="welcome-center-button waves-effect waves-light btn">
-        Dalje
-      </a>
-    </div>
-  </form>
-  
-  <script>
-  
-    $(document).ready(() => {
-  
-      $("#btnsubmit").click(() => {
-        var from = $("#from").val();
-        var to = $("#to").val();
-        console.log("From " + from + " " + to);
-  
-        if (parseInt(from, 10) >= parseInt(to, 10)) {
-          alert("\"Od\" vrijednost ne smije biti veca od \"Do\" vrijednosti ")
-          return;
-        }
-  
-        $("#form-range").submit();
-      });
+  <div class="row center-align no-ui-slider" id="slider"></div>
+
+  <div class="col s6 push-s3">
+    <a id="btnsubmit" href="#" class="welcome-center-button waves-effect waves-light btn">
+      Dalje
+    </a>
+  </div>
+
+  <input type="hidden" name="from" id="from" />
+  <input type="hidden" name="to" id="to" />
+</form>
+
+<script>
+
+  $(document).ready(() => {
+
+    var slider = document.getElementById('slider');
+
+    const unitFormat = wNumb({
+        decimals: 3,
+        thousand: '.',
+        suffix: '<%= unit %>'
+      })
+
+
+    noUiSlider.create(slider, {
+      start: [<%= rangeFrom.value %>, <%= rangeTo.value %>],
+      connect: true,
+      tooltips: true,
+      step: <%= rangeFrom.step %>,
+      range: {
+        'min': <%= rangeFrom.min %>,
+        'max': <%= rangeTo.max %>
+    },
+      format: unitFormat
     });
-  </script>
-  
+
+
+    $("#btnsubmit").click(() => {
+      const sliderValues = slider.noUiSlider.get();
+      
+      $("#from").val(unitFormat.from(sliderValues[0]));
+      $("#to").val(unitFormat.from(sliderValues[1]));
+
+      $("#form-range").submit();
+      // });
+
+    });
+  });
+
+</script>

app/views/unsubscribe.ejs

@@ -0,0 +1,14 @@
+
+<div class="row center-align">
+    <span class="welcome-big-logo">🤙</span>
+</div>
+<div class="row center-align">
+  <div>Uspješno ste se odjavili</div>
+</div>
+<div class="row">
+<div class="col s6 push-s3">
+<a href="<%= nextStep %>" class="welcome-center-button waves-effect waves-light btn">
+  Nova pretraga
+</a>
+</div>
+</div>

development.env

@@ -0,0 +1,5 @@
+  AMAZON_ACCES_KEY_ID=(your-key-here)
+  AMAZON_SECRET_ACCESS_KEY=(your-key-here)
+  AMAZON_REGION=eu-west-1
+  APP_URL=http://localhost:3001
+  SOURCE_EMAIL=info@saburly.com

index.js

@@ -1,5 +1,5 @@
 const welcome = require('./app/controllers/welcome').getWelcome;
-const { getRealEstateTypes, postRealEstateTypes} = require('./app/controllers/realEstateTypes');
+const { getRealEstateTypes, postRealEstateTypes } = require('./app/controllers/realEstateTypes');
 const { getRegion, postRegion } = require('./app/controllers/regions');
 const { getMunicipality, postMunicipality } = require('./app/controllers/municipalities');
 const { getSize, postSize } = require('./app/controllers/sizes');
@@ -9,6 +9,7 @@ const { getQueryReview, postQueryReview } = require('./app/controllers/queryRevi
 const { getQuerySubmit, postQuerySubmit } = require('./app/controllers/querySubmit');
 const { getGoAgain } = require('./app/controllers/goAgain');
 const { getNeighborhood, postNeighborhood } = require('./app/controllers/neighborhoodMap');
+const { getUnsubscribe } = require('./app/controllers/unsubscribe');
 
 let express = require("express");
 const path = require("path");
@@ -106,7 +107,7 @@ app.post("/api/payforalert", (req, res) => {
     }
   };
 
-  tco.checkout.authorize(params, function(error, data) {
+  tco.checkout.authorize(params, function (error, data) {
     if (error) {
       res.send(error.message);
     } else {
@@ -146,6 +147,8 @@ app.post('/pregled/:request_id', postQueryReview);
 app.get('/posalji/:request_id', getQuerySubmit);
 app.post('/posalji/:request_id', postQuerySubmit);
 
+app.get('/odjava/:request_id', getUnsubscribe);
+
 app.get('/ponovo', getGoAgain);
 
 app.use('/assets', express.static('./app/public'));

package-lock.json

@@ -327,9 +327,9 @@
       "dev": true
     },
     "bluebird": {
-      "version": "3.5.3",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz",
-      "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw=="
+      "version": "3.5.5",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz",
+      "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w=="
     },
     "body-parser": {
       "version": "1.18.3",

package.json

@@ -7,6 +7,7 @@
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
+    "scheduler": "node ./app/services/crawlerService.js",
     "migrate": "cd app && npx sequelize db:migrate",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 4 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
@@ -25,6 +26,7 @@
     "2checkout-node": "0.0.1",
     "@sendgrid/mail": "^6.3.1",
     "aws-sdk": "^2.422.0",
+    "bluebird": "^3.5.5",
     "cheerio": "^1.0.0-rc.2",
     "compression": "^1.7.4",
     "dotenv": "^7.0.0",