change price slider options for renting option

Implement renting option - crawler part

See merge request saburly/marketalarm/web!64

.gitignore

@@ -1 +1,3 @@
 node_modules/
+.env
+.idea/

Dockerfile

@@ -1,48 +1,11 @@
-#
-# example Dockerfile for https://docs.docker.com/engine/examples/postgresql_service/
-#
+FROM postgres:11.3
 
-FROM ubuntu:16.04
+ENV POSTGIS_MAJOR 2.4
 
-# Add the PostgreSQL PGP key to verify their Debian packages.
-# It should be the same key as https://www.postgresql.org/media/keys/ACCC4CF8.asc
-RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+RUN apt-get update \
+      && apt-get --assume-yes install software-properties-common postgis\
+      && rm -rf /var/lib/apt/lists/
 
-# Add PostgreSQL's repository. It contains the most recent stable release
-#     of PostgreSQL, ``9.3``.
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN mkdir -p /docker-entrypoint-initdb.d
 
-# Install ``python-software-properties``, ``software-properties-common`` and PostgreSQL 9.3
-#  There are some warnings (in red) that show up during the build. You can hide
-#  them by prefixing each apt-get statement with DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3
-
-# Note: The official Debian and Ubuntu images automatically ``apt-get clean``
-# after each ``apt-get``
-
-# Run the rest of the commands as the ``postgres`` user created by the ``postgres-9.3`` package when it was ``apt-get installed``
-USER postgres
-
-# Create a PostgreSQL role named ``docker`` with ``docker`` as the password and
-# then create a database `docker` owned by the ``docker`` role.
-# Note: here we use ``&&\`` to run commands one after the other - the ``\``
-#       allows the RUN command to span multiple lines.
-RUN    /etc/init.d/postgresql start &&\
-    psql --command "CREATE USER docker WITH SUPERUSER PASSWORD 'docker';" &&\
-    createdb -O docker marketalerts
-
-# Adjust PostgreSQL configuration so that remote connections to the
-# database are possible.
-RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/9.3/main/pg_hba.conf
-
-# And add ``listen_addresses`` to ``/etc/postgresql/9.3/main/postgresql.conf``
-RUN echo "listen_addresses='*'" >> /etc/postgresql/9.3/main/postgresql.conf
-
-# Expose the PostgreSQL port
-EXPOSE 5432
-
-# Add VOLUMEs to allow backup of config, logs and databases
-VOLUME  ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"]
-
-# Set the default command to run when starting the container
-CMD ["/usr/lib/postgresql/9.3/bin/postgres", "-D", "/var/lib/postgresql/9.3/main", "-c", "config_file=/etc/postgresql/9.3/main/postgresql.conf"]
+CMD ["postgres"]

README.md

@@ -1,17 +1,43 @@
-# web
-
+# MarketAlert
 
 The purpose of this project is to build a web application that enables subscribing to notifications when new products are published on various ad based marketplaces. The MVP will be only based on OLX.ba
 
-Create postgres docker image
-docker build -t marketalerts .
+## Setup
 
-Run postgres image with:
-docker run --name pg_test -d -p 5432:5432 marketalerts
+### Setup with npm commands
 
-Run migrations in app folder
-npx sequelize db:migrate
+1. Install packages  
+`npm install`
 
-Run app with: 
-$ npm install 
-$ npm start 
+2. Run setup script  
+`npm run setup`  
+this will create and run postgres image and then execute migrations
+
+3. Run app  
+`npm start` to run app without restart on changes or  
+`npm run start-mon` to run app with automatic restart on code change
+
+
+### Manual setup
+
+1. Create postgres docker image   
+`docker build -t marketalerts .`
+
+2. Run postgres image with  
+`docker run --name pg_marketalerts -d -p 5432:5432 marketalerts`
+
+3. Install packages  
+`npm install`
+
+4. Run migrations from `app` folder  
+`npm run migrate` or `npx sequelize db:migrate`
+
+5. Run app  
+`npm start` or `npm run start-mon` to run app with automatic restart on code change
+
+
+### AWS SES
+
+    - AWS SES credentials are handled with env vratiables
+    - Notification emails are sent in batches of 50, by using SES templates
+    - Make sure that you are using different templates for different envirorments

app/common/enums.js

@@ -0,0 +1,183 @@
+const PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 85000],
+  range: {
+    min: [0],
+    max: [300000]
+  },
+  step: 1000,
+  connect: true
+};
+
+const PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 500],
+  range: {
+    min: [0],
+    max: [2000]
+  },
+  step: 50,
+  connect: true
+};
+
+//This will be used for Flats, Apartments, Houses
+const HOME_SIZE_SLIDER_OPTIONS = {
+  start: [30, 75],
+  range: {
+    min: [0],
+    max: [400]
+  },
+  step: 5,
+  connect: true
+};
+
+const GARDEN_SIZE_SLIDER_OPTIONS = {
+  start: [100, 1000],
+  range: {
+    min: [0],
+    max: [10000]
+  },
+  step: 100,
+  connect: true
+};
+
+const LAND_SIZE_SLIDER_OPTIONS = {
+  start: [5000, 15000],
+  range: {
+    min: [0],
+    max: [100000]
+  },
+  step: 100,
+  connect: true
+};
+const GARAGE_SIZE_SLIDER_OPTIONS = {
+  start: [10, 20],
+  range: {
+    min: [0],
+    max: [150]
+  },
+  step: 2,
+  connect: true
+};
+
+const GARAGE_PRICE_SLIDER_OPTIONS = {
+  start: [2000, 10000],
+  range: {
+    min: [0],
+    max: [100000]
+  },
+  step: 500,
+  connect: true
+};
+
+const AD_TYPE = {
+  AD_TYPE_SALE: {
+    id: 1,
+    stringId: "SALE",
+    title: "Prodaja"
+  },
+  AD_TYPE_RENT: {
+    id: 2,
+    stringId: "RENT",
+    title: "Najam"
+  },
+  AD_TYPE_REQUEST: {
+    id: 3,
+    stringId: "REQUEST",
+    title: "Potražnja"
+  }
+};
+
+const AD_CATEGORY = {
+  ALL: {
+    id: "ALL"
+  },
+  FLAT: {
+    id: "FLAT",
+    title: "Stan",
+    hasGardenSize: false,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+  },
+  HOUSE: {
+    id: "HOUSE",
+    title: "Kuća",
+    hasGardenSize: true,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
+    gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
+  },
+  OFFICE: {
+    id: "OFFICE",
+    title: "Kancelarija",
+    hasGardenSize: false,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+  },
+  LAND: {
+    id: "LAND",
+    title: "Zemljište",
+    hasGardenSize: false,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
+  },
+  APARTMENT: {
+    id: "APARTMENT",
+    title: "Apartman",
+    hasGardenSize: false,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+  },
+  GARAGE: {
+    id: "GARAGE",
+    title: "Garaža",
+    hasGardenSize: false,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: GARAGE_SIZE_SLIDER_OPTIONS
+  },
+  COTTAGE: {
+    id: "COTTAGE",
+    title: "Vikendica",
+    hasGardenSize: true,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
+    gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
+  }
+};
+
+const AD_STATUS = {
+  STATUS_NORMAL: 1,
+  STATUS_RESERVED: 2,
+  STATUS_SOLD: 3,
+  STATUS_DELETED: 4,
+  STATUS_URGENT: 5,
+  STATUS_DISCOUNTED: 6
+};
+
+const AD_AGENCY = {
+  OLX: "OLX",
+  RENTAL: "RENTAL",
+  PROSTOR: "PROSTOR",
+  AKTIDO: "AKTIDO"
+};
+
+const CRAWLER_AD_TYPE = {
+  NONE: 0,
+  ALL: 1,
+  ONLY_SELL: 2,
+  ONLY_RENT: 3,
+  ONLY_REQUEST: 4
+};
+
+module.exports = {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_STATUS,
+  AD_AGENCY,
+  CRAWLER_AD_TYPE
+};

app/config/appConfig.js

@@ -0,0 +1,43 @@
+"use strict";
+require("dotenv").config({ path: __dirname + "/./../../.env" });
+
+const APP_PORT = process.env.PORT || 5000;
+const APP_BASE_URL = process.env.APP_BASE_URL || "http://localhost";
+
+const APP_URL =
+  process.env.NODE_ENV && process.env.NODE_ENV === "production"
+    ? process.env.APP_URL || "http://market-alarm"
+    : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
+
+const DEFAULT_TIMEZONE = "Europe/Sarajevo";
+
+const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
+const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
+
+const AWS_EMAIL_CONFIG = {
+  REGION: process.env.AWS_REGION || "",
+  CREDENTIALS: {
+    ACCESS_KEY_ID: process.env.AWS_KEY_ID || "",
+    SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ""
+  },
+  SOURCE_EMAIL: process.env.SOURCE_EMAIL || ""
+};
+
+const MAX_REAL_ESTATES_IN_EMAIL =
+  parseInt(process.env.MAX_REAL_ESTATES_IN_EMAIL) || 10;
+const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
+  parseInt(process.env.MAX_REAL_ESTATES_IN_FIRST_EMAIL) || 5;
+
+const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
+
+module.exports = {
+  APP_PORT,
+  APP_URL,
+  DEFAULT_TIMEZONE,
+  CRAWLER_INTERVAL,
+  STOP_CRAWLER,
+  AWS_EMAIL_CONFIG,
+  MAX_REAL_ESTATES_IN_EMAIL,
+  MAX_REAL_ESTATES_IN_FIRST_EMAIL,
+  PRINT_CRAWLER_DEBUG
+};

app/config/config.json

@@ -7,9 +7,9 @@
     "dialect": "postgres"
   },
   "test": {
-    "use_env_variable": "JAWSDB_URL"
+    "use_env_variable": "DATABASE_URL"
   },
   "production": {
-    "use_env_variable": "JAWSDB_URL"
+    "use_env_variable": "DATABASE_URL"
   }
 }

app/controllers/city.js

@@ -1,27 +0,0 @@
-const db = require('../models/index');
-const  { currentRERequest } = require('../helpers/url'); 
-const { regions } = require('../helpers/codes');
-
-const cities = regions();
-
-
-const getCity = (req,res) => {
-  const nextStep = req.query.nextStep || '/';
-  res.render('city', { 
-         nextStep, 
-         cities  
-  });
-}
-
-const postCity = async (req, res) => {
-  const request = await currentRERequest(req);
-  const nextStep = req.query.nextStep || `/mjesto/${request.uniqueId}`;
-  request.city = req.body.city;
-  await request.save();
-  res.redirect(nextStep)
-}
-
-module.exports = { 
-  getCity,
-  postCity
-};

app/controllers/goAgain.js

@@ -0,0 +1,8 @@
+const getGoAgain = async (req, res) => {
+  const title = "Uspjeh!";
+  res.render("goAgain", { title });
+};
+
+module.exports = {
+  getGoAgain
+};

app/controllers/location.js

@@ -0,0 +1,57 @@
+const { currentSearchRequest } = require("../helpers/url");
+
+const getLocation = async (req, res) => {
+  const title = "Odaberite lokaciju";
+  const nextStep = req.query.nextStep || "/";
+
+  res.render("location", {
+    nextStep,
+    title
+  });
+};
+
+const postLocation = async (req, res) => {
+  let searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const northWest = [req.body.west, req.body.north];
+  const northEast = [req.body.east, req.body.north];
+  const southEast = [req.body.east, req.body.south];
+  const southWest = [req.body.west, req.body.south];
+
+  const locationInputValue =
+    req.body.locationInput && req.body.locationInput.length > 0
+      ? req.body.locationInput
+      : null;
+
+  searchRequest.areaToSearch = {
+    type: "Polygon",
+    coordinates: [[northWest, northEast, southEast, southWest, northWest]],
+    crs: { type: "name", properties: { name: "EPSG:4326" } }
+  };
+
+  let locationInputData;
+  if (req.body.locationInputData) {
+    try {
+      locationInputData = JSON.parse(req.body.locationInputData);
+    } catch (e) {
+      locationInputData = null;
+    }
+  }
+
+  await searchRequest.save();
+
+  const nextStepPage = req.query.nextStep || "filteri";
+  const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getLocation,
+  postLocation
+};

app/controllers/neighborhoods.js

@@ -1,25 +0,0 @@
-const db = require('../models/index');
-const  { currentRERequest } = require('../helpers/url'); 
-const { places } = require('../helpers/codes');
-
-const getNeighborhood = async (req,res) => {
-  let request = await currentRERequest(req);
-  const neighborhoods = places(request.city);
-  const nextStep = req.query.nextStep || '/';
-  res.render('neighborhood', { 
-         nextStep, 
-         neighborhoods  
-  });
-}
-
-const postgNeighborhood = async (req, res) => {
-  let request = await currentRERequest(req);
-  request.neighborhood = req.body.neighborhood;
-  await request.save();
-  res.send("Result is " + JSON.stringify(request)); 
-}
-
-module.exports = { 
-  getNeighborhood,
-  postgNeighborhood
-};

app/controllers/queryReview.js

@@ -0,0 +1,179 @@
+const { currentSearchRequest } = require("../helpers/url");
+const { isValidEmail } = require("../helpers/email");
+const {
+  notifyForNewSearchRequest
+} = require("../services/notificationService");
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
+
+const getQueryReviewData = searchRequest => {
+  const {
+    id,
+    adType,
+    realEstateType,
+    sizeMin,
+    sizeMax,
+    gardenSizeMin,
+    gardenSizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest.dataValues;
+
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+  const enableGardenSizeEdit = realEstateTypeObject
+    ? realEstateTypeObject.hasGardenSize
+    : false;
+
+  let adTypeTitle = "";
+  switch (adType) {
+    case AD_TYPE.AD_TYPE_SALE.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
+      break;
+    case AD_TYPE.AD_TYPE_RENT.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
+      break;
+    default:
+      adTypeTitle = "-";
+      break;
+  }
+
+  const realEstateTypeTitle = realEstateTypeObject
+    ? `[${adTypeTitle}] ${realEstateTypeObject.title}`
+    : "-";
+
+  const locationTitle = "Promjenite lokaciju";
+  const sizeTitle = `${sizeMin} - ${sizeMax} m2`;
+  const gardenSizeTitle = enableGardenSizeEdit
+    ? `${gardenSizeMin} - ${gardenSizeMax} m2`
+    : "-";
+  const priceTitle = `${priceMin} - ${priceMax} KM`;
+
+  return [
+    {
+      id: "realEstateType",
+      title: realEstateTypeTitle,
+      url: `/vrstanekretnine/${id}?nextStep=filteri`
+    },
+    {
+      id: "location",
+      title: locationTitle,
+      url: `/lokacija/${id}?nextStep=pregled`
+    },
+    {
+      id: "size",
+      title: sizeTitle,
+      url: `/filteri/${id}?nextStep=pregled`
+    },
+    {
+      id: "gardenSize",
+      title: gardenSizeTitle,
+      url: enableGardenSizeEdit ? `/filteri/${id}?nextStep=pregled` : ""
+    },
+    {
+      id: "price",
+      title: priceTitle,
+      url: `/filteri/${id}?nextStep=pregled`
+    }
+  ].filter(data => data.title != "-");
+};
+
+const getQueryReview = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const title = "Da li je ovo to što ste tražili ?";
+  const nextStep = req.query.nextStep;
+  const error = req.query.error;
+  const queryReviewData = getQueryReviewData(searchRequest);
+  const email = searchRequest.email;
+
+  res.render("queryReview", {
+    nextStep,
+    queryReviewData,
+    title,
+    email,
+    error
+  });
+};
+
+const postQueryReview = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    return null;
+  }
+
+  const nextStep = req.query.nextStep || "/ponovo";
+  const emailInput = req.body.email;
+  const emailConfirmInput = req.body.confirmEmail;
+  const title = "Da li je ovo to što ste tražili ?";
+  const queryReviewData = getQueryReviewData(searchRequest);
+
+  if (emailInput !== emailConfirmInput) {
+    const error = "Greška ! Unešeni emailovi nisu isti";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewData,
+      email: ""
+    });
+    return;
+  }
+
+  if (!isValidEmail(emailInput)) {
+    const error = "Greška ! Unesite validan email";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewData,
+      email: ""
+    });
+    return;
+  }
+
+  searchRequest.email = emailInput;
+  searchRequest.subscribed = true;
+
+  try {
+    await searchRequest.save();
+  } catch (e) {
+    console.log("[ERROR] Failed to save search request !", e);
+    console.log("Search request : ", searchRequest);
+    const error =
+      "Greška ! Nismo uspjeli kreirati zahtjev za Vašu pretragu. Molimo pokuštajte ponovo";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewData,
+      email: ""
+    });
+    return;
+  }
+
+  try {
+    await notifyForNewSearchRequest(searchRequest);
+  } catch (e) {
+    console.log("[ERROR] Failed to send initial welcome email", e);
+    console.log("Search request : ", searchRequest);
+
+    const error =
+      "Greška ! Nismo uspjeli poslati email na Vašu adresu, pokušajte sa drugom email adresom";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewData,
+      email: ""
+    });
+    return;
+  }
+
+  res.redirect(nextStep);
+};
+
+module.exports = {
+  getQueryReview,
+  postQueryReview
+};

app/controllers/realEstateFilters.js

@@ -0,0 +1,109 @@
+const { currentSearchRequest } = require("../helpers/url");
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
+
+const getFilters = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const title = "Filteri za pretraživanje";
+
+  const {
+    adType,
+    realEstateType,
+    priceMin,
+    priceMax,
+    sizeMin,
+    sizeMax,
+    gardenSizeMin,
+    gardenSizeMax
+  } = searchRequest;
+  const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
+
+  const {
+    hasGardenSize,
+    priceSliderOptionsSale,
+    priceSliderOptionsRent,
+    sizeSliderOptions,
+    gardenSizeSliderOptions
+  } = category;
+
+  let priceSliderOptions;
+  if (adType === AD_TYPE.AD_TYPE_SALE.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsSale);
+  } else if (adType === AD_TYPE.AD_TYPE_RENT.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsRent);
+  } else {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  if (priceMin || priceMax) {
+    priceSliderOptions.start = [priceMin, priceMax];
+  }
+
+  if (sizeMin || sizeMax) {
+    sizeSliderOptions.start = [sizeMin, sizeMax];
+  }
+
+  if (gardenSizeSliderOptions && (gardenSizeMin || gardenSizeMax)) {
+    gardenSizeSliderOptions.start = [gardenSizeMin, gardenSizeMax];
+  }
+
+  res.render("realEstateFilters", {
+    title,
+    hasGardenSize,
+    priceSliderOptions: JSON.stringify(priceSliderOptions),
+    sizeSliderOptions: JSON.stringify(sizeSliderOptions),
+    gardenSizeSliderOptions: JSON.stringify(gardenSizeSliderOptions)
+  });
+};
+
+const postFilters = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const nextStepPage = req.query.nextStep || "pregled";
+  const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+
+  const priceMin = parseInt(req.body.priceMin) || 0;
+  const priceMax = parseInt(req.body.priceMax) || 0;
+  const sizeMin = parseInt(req.body.sizeMin) || 0;
+  const sizeMax = parseInt(req.body.sizeMax) || 0;
+
+  //TODO: Filter validation
+
+  searchRequest.priceMin = priceMin;
+  searchRequest.priceMax = priceMax;
+  searchRequest.sizeMin = sizeMin;
+  searchRequest.sizeMax = sizeMax;
+
+  if (
+    req.body.gardenSizeMin !== undefined &&
+    req.body.gardenSizeMax !== undefined
+  ) {
+    const gardenSizeMin = parseInt(req.body.gardenSizeMin);
+    const gardenSizeMax = parseInt(req.body.gardenSizeMax);
+
+    //TODO: Filter validation
+
+    searchRequest.gardenSizeMin = gardenSizeMin;
+    searchRequest.gardenSizeMax = gardenSizeMax;
+  }
+
+  await searchRequest.save();
+
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getFilters,
+  postFilters
+};

app/controllers/realEstateTypes.js

@@ -0,0 +1,81 @@
+const { currentSearchRequest } = require("../helpers/url");
+const { createSearchRequest } = require("../helpers/db/searchRequest");
+
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
+
+const getRealEstateTypes = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  const title = "Koju nekretninu tražite?";
+  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
+  if (
+    searchRequest &&
+    searchRequest.adType &&
+    searchRequest.adType === AD_TYPE.AD_TYPE_RENT.stringId
+  ) {
+    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
+  }
+  const realEstateTypes = Object.keys(AD_CATEGORY)
+    .map(category => AD_CATEGORY[category])
+    .filter(category => category.title);
+
+  res.render("realEstateType", {
+    selectedAdType,
+    realEstateTypes,
+    title,
+    AD_TYPE
+  });
+};
+
+const postRealEstateTypes = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  const adType = parseInt(req.body.adType);
+
+  const adTypeStringIds = {
+    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
+    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
+  };
+
+  const adTypeStringId =
+    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
+
+  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
+    category => !!AD_CATEGORY[category].title
+  );
+
+  const selectedRealEstateType = req.body.realEstateType || null;
+  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const nextStepPage = req.query.nextStep || "lokacija";
+
+  let nextStepUrl = "";
+  if (searchRequest && searchRequest.id) {
+    nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+    searchRequest.adType = adTypeStringId;
+    searchRequest.realEstateType = selectedRealEstateType;
+
+    await searchRequest.save();
+  } else {
+    try {
+      const newSearchRequest = await createSearchRequest({
+        adType: adTypeStringId,
+        realEstateType: selectedRealEstateType
+      });
+
+      nextStepUrl = `/${nextStepPage}/${newSearchRequest.id}`;
+    } catch (error) {
+      console.log(error);
+      nextStepUrl = `/`;
+    }
+  }
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getRealEstateTypes,
+  postRealEstateTypes
+};

app/controllers/realEstates.js

@@ -0,0 +1,16 @@
+"use strict";
+const {
+  findRealEstatesForSearchRequest
+} = require("../helpers/db/searchRequestMatch");
+
+const getRealEstates = async (req, res) => {
+  const searchRequestId = req.params["searchRequestId"] || "";
+  const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
+
+  const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
+  res.render("realEstates", { realEstates, title });
+};
+
+module.exports = {
+  getRealEstates
+};

app/controllers/real_estate_types.js

@@ -1,34 +0,0 @@
-const db = require('../models/index');
-
-const realEstateTypes = [
-  { ime: "Kuća", id: "kuca" }, 
-  { ime: "Stan", id: "stan" }, 
-  { ime: "Vikendica", id: "vikendica" }
-];
-
-
-const getRealEstateTypes = (req,res) => {
-  const nextStep = req.query.nextStep; 
-  res.render('real_estate_type', { 
-         nextStep, 
-         realEstateTypes: realEstateTypes 
-  });
-}
-
-const postRealEstateTypes = (req, res) => {
-  let nextStep = req.query.nextStep;
-  db.RealEstateRequest.create({
-    realEstateType: req.body.realestatetype
-  }).then( (result) => {
-    nextStep = nextStep || `/grad/${result.uniqueId}`;
-    res.redirect(nextStep); 
-  }).catch( (e) => {
-   res.send(e);
-  }); 
-  
-}
-
-module.exports = { 
-  getRealEstateTypes,
-  postRealEstateTypes
-};

app/controllers/redirect.js

@@ -0,0 +1,33 @@
+const { getRealEstateById } = require("../helpers/db/realEstate");
+
+const getRedirect = async (req, res) => {
+  const id = req.params.id || null;
+  let error = false;
+  let redirectUrl = undefined;
+  if (!id) {
+    error = true;
+  } else {
+    try {
+      const realEstate = await getRealEstateById(id);
+      if (!realEstate) {
+        error = true;
+      } else {
+        redirectUrl = realEstate.url;
+      }
+    } catch (e) {
+      error = true;
+    }
+  }
+
+  if (error) {
+    const title = "";
+    res.render("notFound", { title });
+  } else {
+    const title = "Preusmjeravanje";
+    res.render("redirect", { title, redirectUrl });
+  }
+};
+
+module.exports = {
+  getRedirect
+};

app/controllers/unsubscribe.js

@@ -0,0 +1,20 @@
+const { currentSearchRequest } = require("../helpers/url");
+
+const getUnsubscribe = async (req, res) => {
+  const title = "Uspješno ste se odjavili";
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  searchRequest.subscribed = false;
+  await searchRequest.save();
+
+  res.render("unsubscribe", { nextStep: "/vrstanekretnine", title });
+};
+
+module.exports = {
+  getUnsubscribe
+};

app/controllers/welcome.js

@@ -1,7 +1,7 @@
-const getWelcome = (req,res) => {
-  res.render('welcome', { nextStep: '/vrstanekretnine' } );
-}
+const getWelcome = (req, res) => {
+  res.render("welcome", { nextStep: "/vrstanekretnine", title: false });
+};
 
-module.exports = { 
+module.exports = {
   getWelcome
 };

app/crawler/crawl.js

@@ -0,0 +1,81 @@
+"use strict";
+/*
+   Entry point for crawling functionality
+   All communication between crawlers and savers is here
+   All environment specific configuration is read here and
+   passed to the crawlers and savers.
+*/
+const OlxCrawler = require("./specificCrawlers/olx");
+const RentalCrawler = require("./specificCrawlers/rental");
+const ProstorCrawler = require("./specificCrawlers/prostor");
+const AktidoCrawler = require("./specificCrawlers/aktido");
+
+const {
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG
+} = require("./crawlerConfig");
+const PostgresSaver = require("./savers/postgres");
+
+async function crawlAll() {
+  const postgresSaver = new PostgresSaver();
+
+  const crawlers = [
+    new OlxCrawler(
+      [postgresSaver],
+      OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
+      OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
+      OLX_CONFIG.OLX_MAX_PAGES,
+      OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+      OLX_CONFIG.OLX_IGNORED_USERNAMES,
+      OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
+    ),
+    new RentalCrawler(
+      [postgresSaver],
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
+      RENTAL_CONFIG.RENTAL_MAX_PAGES,
+      RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
+      RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
+      RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
+    ),
+    new ProstorCrawler(
+      [postgresSaver],
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
+      PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
+      PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
+      PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
+      PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
+    ),
+    new AktidoCrawler(
+      [postgresSaver],
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
+      AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
+      AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
+      AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
+      AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
+    )
+  ];
+
+  const newRealEstates = [];
+
+  for (const crawler of crawlers) {
+    try {
+      const newRealEstatesFromSingleCrawler = await crawler.crawl();
+      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
+        newRealEstates.push(...newRealEstatesFromSingleCrawler);
+      }
+    } catch (e) {
+      console.log("Error crawling. Trying next crawler! ", e);
+    }
+  }
+
+  return newRealEstates;
+}
+
+module.exports = {
+  crawlAll
+};

app/crawler/crawlerConfig.js

@@ -0,0 +1,14 @@
+"use strict";
+require("dotenv").config({ path: __dirname + "/./../../.env" });
+
+const OLX_CONFIG = require("./specificConfigs/olx");
+const RENTAL_CONFIG = require("./specificConfigs/rental");
+const PROSTOR_CONFIG = require("./specificConfigs/prostor");
+const AKTIDO_CONFIG = require("./specificConfigs/aktido");
+
+module.exports = {
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG
+};

app/crawler/npmCrawl.js

@@ -0,0 +1,5 @@
+const { crawlAll } = require("./crawl");
+
+(async () => {
+  await crawlAll();
+})();

app/crawler/savers/postgres.js

@@ -0,0 +1,47 @@
+const moment = require("moment");
+
+const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
+
+class PostgresSaver {
+  connect() {
+    //TODO: It seems we never worry about open/close connection with Sequelize ?
+    //TODO: Check if postgres is ready
+    return true;
+  }
+
+  async save(results) {
+    const savedRecords = await bulkUpsertRealEstates(results);
+
+    if (Array.isArray(savedRecords)) {
+      const newRealEstates = [];
+      const existingRealEstates = [];
+
+      for (const savedRecord of savedRecords) {
+        const { createdAt, updatedAt } = savedRecord;
+
+        const createdAtMoment = moment.utc(createdAt);
+        const updatedAtMoment = moment.utc(updatedAt);
+
+        if (createdAtMoment.isSame(updatedAtMoment, "second")) {
+          newRealEstates.push(savedRecord);
+        } else {
+          existingRealEstates.push(savedRecord);
+        }
+      }
+
+      return {
+        newRecords: newRealEstates,
+        existingRecords: existingRealEstates
+      };
+    } else {
+      throw { message: "[POSTGRES] Failed to save records" };
+    }
+  }
+
+  close() {
+    //TODO: It seems we never worry about open/close connection with Sequelize ?
+    return true;
+  }
+}
+
+module.exports = PostgresSaver;

app/crawler/specificConfigs/aktido.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const aktidoCrawlerAdType =
+  process.env.AKTIDO_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.AKTIDO_CRAWLER_AD_TYPE]
+    : null;
+
+const aktidoParsedCrawlerAdCategories =
+  process.env.AKTIDO_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.AKTIDO_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const aktidoIgnoredUsernames = [];
+
+const transformedAktidoCrawlerAdCategories = aktidoParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  AKTIDO_MAX_PAGES: parseInt(process.env.AKTIDO_MAX_PAGES) || 500,
+  AKTIDO_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.AKTIDO_MAX_RESULTS_PER_PAGE) || 50,
+  AKTIDO_CRAWLER_AD_TYPE: aktidoCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
+  AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
+  AKTIDO_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
+  AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
+};

app/crawler/specificConfigs/olx.js

@@ -0,0 +1,39 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const olxCrawlerAdType =
+  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
+    : null;
+
+const olxParsedCrawlerAdCategories =
+  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const olxIgnoredUsernames =
+  process.env.OLX_IGNORED_USERNAMES !== undefined
+    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
+        username.trim()
+      )
+    : [];
+
+const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
+  OLX_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
+  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
+  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
+  OLX_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
+  OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
+};

app/crawler/specificConfigs/prostor.js

@@ -0,0 +1,33 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const prostorCrawlerAdType =
+  process.env.PROSTOR_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.PROSTOR_CRAWLER_AD_TYPE]
+    : null;
+
+const prostorParsedCrawlerAdCategories =
+  process.env.PROSTOR_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.PROSTOR_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const prostorIgnoredUsernames = [];
+
+const transformedProstorCrawlerAdCategories = prostorParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  PROSTOR_MAX_PAGES: parseInt(process.env.PROSTOR_MAX_PAGES) || 100,
+  PROSTOR_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 5000,
+  PROSTOR_CRAWLER_AD_TYPE: prostorCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  PROSTOR_CRAWLER_AD_CATEGORIES: transformedProstorCrawlerAdCategories,
+  PROSTOR_IGNORED_USERNAMES: prostorIgnoredUsernames || [],
+  PROSTOR_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.PROSTOR_DELAY_BETWEEN_PAGES) || 1000
+};

app/crawler/specificConfigs/rental.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const rentalCrawlerAdType =
+  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
+    : null;
+
+const rentalParsedCrawlerAdCategories =
+  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const rentalIgnoredUsernames = [];
+
+const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
+  RENTAL_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
+  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
+  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
+  RENTAL_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000,
+  RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL)
+};

app/crawler/specificCrawlers/aktido.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const AKTIDO_ENUMS = {
+  AKTIDO_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  AKTIDO_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  AKTIDO_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
+
+class AktidoCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.aktido.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !AKTIDO_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = AKTIDO_ENUMS.AKTIDO_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = AKTIDO_ENUMS.AKTIDO_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[AKTIDO] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".moreInfo")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[AKTIDO] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        const aktidoId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromAktidoId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromAktidoActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          AKTIDO_ENUMS.AKTIDO_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          AKTIDO_ENUMS.AKTIDO_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const adStatus = AD_STATUS.STATUS_NORMAL;
+
+        const data = {
+          url,
+          agencyObjectId: aktidoId,
+          originAgencyName: AD_AGENCY.AKTIDO,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString()
+        };
+
+        return data;
+      } else {
+        console.log("[AKTIDO] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromAktidoId(aktidoCategoryId) {
+    switch (aktidoCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromAktidoActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = AktidoCrawler;

app/crawler/specificCrawlers/olx.js

@@ -0,0 +1,577 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const OLX_ENUMS = {
+  OLX_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje",
+    [CRAWLER_AD_TYPE.ONLY_REQUEST]: "&vrsta=samopotraznja"
+  },
+  OLX_AD_CATEGORY: {
+    [AD_CATEGORY.FLAT.id]: "&kategorija=23",
+    [AD_CATEGORY.HOUSE.id]: "&kategorija=24",
+    [AD_CATEGORY.LAND.id]: "&kategorija=29",
+    [AD_CATEGORY.OFFICE.id]: "&kategorija=25",
+    [AD_CATEGORY.APARTMENT.id]: "&kategorija=27",
+    [AD_CATEGORY.GARAGE.id]: "&kategorija=30",
+    [AD_CATEGORY.COTTAGE.id]: "&kategorija=26"
+  },
+  MAX_DETAIL_FIELDS: 30,
+  OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
+  OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
+};
+
+const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
+
+class OlxCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.ignoredUsernames = ignoredUsernames;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords, existingRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            for (const existingRecord of existingRecords) {
+              const { publishedDate, renewedDate } = existingRecord;
+
+              const publishedDateMoment = moment.utc(publishedDate);
+              const renewedDateMoment = moment.utc(renewedDate);
+
+              const stopCrawlingThisCategory = publishedDateMoment.isSame(
+                renewedDateMoment,
+                "minute"
+              );
+
+              if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
+                generatorsToRemove[index] = true;
+                // console.log("\tGenerator ", index + 1, "has no more new ads");
+                break;
+              }
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[OLX] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $("#rezultatipretrage")
+        .find(".listitem")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+      let status = AD_STATUS.STATUS_NORMAL;
+
+      const propertySelectors = {
+        username:
+          "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
+        title: "#naslovartikla",
+        descriptions: ".artikal_detaljniopis_tekst",
+        category:
+          "#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
+      };
+
+      const username = $(propertySelectors.username)
+        .text()
+        .trim();
+      if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
+        return null;
+      }
+
+      const title = $(propertySelectors.title)
+        .text()
+        .trim();
+      const descriptions = $(propertySelectors.descriptions);
+      const category = $(propertySelectors.category)
+        .text()
+        .trim();
+
+      //====== PRICE DETECTION AND EXTRACTION =====
+      let price = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      const urgentPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
+      )
+        .text()
+        .trim();
+
+      if (normalPriceValue && normalPriceValue.length > 0) {
+        price = normalPriceValue;
+        if (
+          $("#pc > p.n")
+            .text()
+            .indexOf("Hitna") !== -1
+        ) {
+          status = AD_STATUS.STATUS_URGENT;
+        } else {
+          status = AD_STATUS.STATUS_NORMAL;
+        }
+      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.split("KM");
+        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
+        if (priceValues.length > 1) {
+          price = priceValues[1].trim();
+          status = AD_STATUS.STATUS_DISCOUNTED;
+        } else {
+          throw { message: "Can't find urgent price" };
+        }
+      } else {
+        throw {
+          message: "Can't find price (it is not normal nor urgent price ?)"
+        };
+      }
+
+      //====== OTHER AD INFORMATION ===============
+      let adType = null;
+      let olxId = null;
+
+      let otherInformationDivId;
+      //We need to locate DIV ID where other information are stored
+      for (let possibleId = 10; possibleId <= 20; possibleId++) {
+        const adTypeFieldTitle = $(
+          `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
+        )
+          .text()
+          .trim();
+
+        if (adTypeFieldTitle === "Vrsta oglasa") {
+          otherInformationDivId = possibleId;
+          break;
+        }
+      }
+
+      if (!otherInformationDivId) {
+        throw { message: "Other information DIV could not be found" };
+      }
+
+      const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
+      const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
+      const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
+
+      const publishedDate = $(publishedDateValueSelector)
+        .text()
+        .trim();
+
+      const publishedDateMoment = moment.tz(
+        publishedDate,
+        OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
+        DEFAULT_TIMEZONE
+      );
+
+      if (!publishedDateMoment.isValid()) {
+        throw { message: "Invalid published date ! Check parsing format" };
+      }
+
+      const renewedDate = $(renewedDateFullValueSelector)
+        .data("content")
+        .trim();
+
+      const renewedDateMoment = moment.tz(
+        renewedDate,
+        OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
+        DEFAULT_TIMEZONE
+      );
+
+      if (!renewedDateMoment) {
+        throw {
+          message:
+            "Invalid renewed date ! Check how parser parsed renewed date text"
+        };
+      }
+
+      adType = $(
+        `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
+      )
+        .text()
+        .trim();
+      const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
+        .text()
+        .trim();
+      olxId = $(`${olxIdFieldSelector} > div.df2`)
+        .text()
+        .trim();
+
+      if (olxIdFieldTitle !== "OLX ID") {
+        throw { message: "Cannot find correct OLX ID" };
+      }
+      //===========================================
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area = null;
+      let gardenSize = null;
+
+      let fieldIndex = 1;
+      do {
+        const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
+        const fieldTitleSelector = `${fieldSelector} > div.df1`;
+        const fieldValueSelector = `${fieldSelector} > div.df2`;
+
+        const fieldTitle = $(fieldTitleSelector)
+          .text()
+          .trim();
+        const fieldValue = $(fieldValueSelector)
+          .text()
+          .trim();
+
+        switch (fieldTitle) {
+          case "Kvadrata":
+            area = fieldValue;
+            break;
+          case "Okućnica (kvadratura)":
+            gardenSize = fieldValue;
+            break;
+        }
+
+        if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
+          break;
+        }
+      } while (true);
+      //===========================================
+
+      //====== UNUSED FIELDS FOR NOW ==============
+      const time = $("time").attr("datetime");
+      const numberOfViews = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(18) > div:nth-child(6) > div.df2"
+      )
+        .text()
+        .trim();
+      //===========================================
+
+      //=========================================
+      const parsedCategory = this.getAdCategoryId(category);
+      if (!parsedCategory) {
+        throw { message: `Unknown ad category [${category}]` };
+      }
+
+      const parsedAdType = this.getAdTypeId(adType);
+      if (!parsedAdType) {
+        throw { message: "Unknown ad type" };
+      }
+
+      const parsedArea = this.parseArea(area) || null;
+      const parsedGardenSize = this.parseArea(gardenSize) || null;
+      const parsedPrice = this.parsePrice(price) || null;
+
+      const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+      const locationLatLngMatches = latLngRegex.exec(body);
+
+      let locationLat = null;
+      let locationLong = null;
+      if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
+        locationLat = parseFloat(locationLatLngMatches[1]) || null;
+        locationLong = parseFloat(locationLatLngMatches[2]) || null;
+      }
+
+      const data = {
+        url,
+        agencyObjectId: olxId,
+        originAgencyName: AD_AGENCY.OLX,
+        realEstateType: parsedCategory,
+        adType: parsedAdType,
+        title,
+        price: parsedPrice,
+        area: parsedArea,
+        gardenSize: parsedGardenSize,
+        shortDescription: descriptions
+          .first()
+          .text()
+          .trim(),
+        longDescription: descriptions
+          .last()
+          .text()
+          .trim(),
+        streetNumber: 0,
+        streetName: "",
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat,
+        locationLong,
+        adStatus: status,
+        publishedDate: publishedDateMoment.toISOString(),
+        renewedDate: renewedDateMoment.toISOString()
+      };
+
+      return data;
+    } catch (e) {
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "Stanovi":
+        return AD_CATEGORY.FLAT.id;
+      case "Zemljišta":
+        return AD_CATEGORY.LAND.id;
+      case "Kuće":
+        return AD_CATEGORY.HOUSE.id;
+      case "Poslovni prostori":
+        return AD_CATEGORY.OFFICE.id;
+      case "Apartmani":
+        return AD_CATEGORY.APARTMENT.id;
+      case "Garaže":
+        return AD_CATEGORY.GARAGE.id;
+      case "Vikendice":
+        return AD_CATEGORY.COTTAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "Prodaja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "Izdavanje":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      case "Potražnja":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  parseArea(areaText) {
+    if (!areaText) {
+      return NaN;
+    }
+    const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
+    const textWithOnlyOneDecimalDot = areaText
+      .replace(",", ".")
+      .replace(removeDotsExceptLastOneRegex, "");
+
+    return parseFloat(textWithOnlyOneDecimalDot);
+  }
+
+  parsePrice(priceText) {
+    if (!priceText) {
+      return NaN;
+    }
+    const formattedPriceText = priceText.replace(".", "").replace(",", ".");
+    return parseFloat(formattedPriceText);
+  }
+
+  parseRenewedDate(renewedDateText) {
+    const currentMoment = moment.tz(DEFAULT_TIMEZONE);
+
+    if (renewedDateText.includes("Prije mjesec dana")) {
+      return currentMoment.add(-1, "month");
+    }
+
+    if (renewedDateText.includes("Jučer")) {
+      return currentMoment.add(-1, "day");
+    }
+
+    if (renewedDateText.includes("Prije sat")) {
+      return currentMoment.add(-1, "hour");
+    }
+
+    if (renewedDateText.includes("dan")) {
+      // format for this case should be "Prije N dana" or "Prije N dan"
+      const dateParts = renewedDateText.split(" ");
+      if (dateParts[0] === "Prije") {
+        const numberOfDays = parseInt(dateParts[1]);
+        return currentMoment.add(-1 * numberOfDays, "days");
+      } else {
+        return undefined;
+      }
+    }
+
+    if (renewedDateText.includes("sat")) {
+      const dateParts = renewedDateText.split(" ");
+      const parsedHours =
+        dateParts && dateParts.length > 2 ? parseInt(dateParts[1]) : undefined;
+      if (!parsedHours) {
+        return undefined;
+      }
+      return currentMoment.add(-1 * parsedHours, "hours");
+    }
+
+    const todayVariations = ["min", "sekund", "maloprije"];
+    for (const todayVariation of todayVariations) {
+      if (renewedDateText.includes(todayVariation)) {
+        return currentMoment;
+      }
+    }
+
+    const renewedDateMoment = moment.tz(
+      renewedDateText,
+      OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
+      DEFAULT_TIMEZONE
+    );
+
+    return renewedDateMoment.isValid() ? renewedDateMoment : undefined;
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = OlxCrawler;

app/crawler/specificCrawlers/prostor.js

@@ -0,0 +1,252 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const { PRINT_CRAWLER_DEBUG } = require("../../config/appConfig");
+
+const PROSTOR_ENUMS = {
+  PROSTOR_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&action=0",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&action=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&action=2"
+  },
+  PROSTOR_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "&type=7",
+    [AD_CATEGORY.HOUSE.id]: "&type=8",
+    [AD_CATEGORY.LAND.id]: "&type=10",
+    [AD_CATEGORY.OFFICE.id]: "&type=9",
+    [AD_CATEGORY.APARTMENT.id]: "&type=11",
+    [AD_CATEGORY.GARAGE.id]: "&type=14"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  PROSTOR_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  PROSTOR_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+class ProstorCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://prostor.ba/pretraga";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      for (const adCategory of crawlAdCategories) {
+        const urlAdTypePart =
+          PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
+        const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
+        if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+          const urlPageToCrawl = `${this.baseUrl}?remove_sold=1${urlAdTypePart}${urlCategoryPart}`;
+          const singleCategoryResults = await this.extractRealEstates(
+            urlPageToCrawl
+          );
+
+          const resultsSubset = singleCategoryResults.slice(
+            0,
+            this.maxResultsPerPage
+          );
+
+          const saveResults = await this.saveCrawledResults(resultsSubset);
+          const { newRecords } = saveResults;
+          newRealEstates.push(...newRecords);
+        }
+      }
+    }
+    return newRealEstates;
+  }
+
+  async extractRealEstates(url) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[PROSTOR] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      const scriptElement = $(
+        "body > div > div.container-fluid > script:nth-child(7)"
+      );
+
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        const scriptData = scriptElement[0].children[0].data;
+
+        try {
+          // script element data contains JS code and we need to extract only data for realEstates
+          // data string starts with : var map; var markers = [{"r ...
+          // so we remove first 23 characters
+          //
+          // real estate JSON data ends with      ...}, ]; map = new...
+          // so we need to find index of that substring to know where to stop
+          // we will NOT include trailing comma because it breaks JSON parse, so we have to close ] bracket manually
+
+          const jsonEndIndex = scriptData.indexOf(", ]; map = new");
+          if (jsonEndIndex > -1) {
+            const jsonData = scriptData.substring(23, jsonEndIndex) + "]";
+            const realEstates = JSON.parse(jsonData);
+
+            const transformedRealEstates = [];
+
+            for (const realEstate of realEstates) {
+              const transformedRealEstate = ProstorCrawler.transformRealEstateData(
+                realEstate
+              );
+              if (transformedRealEstate) {
+                transformedRealEstates.push(transformedRealEstate);
+              }
+            }
+
+            return transformedRealEstates;
+          } else {
+            throw {
+              message: "Something is wrong with JSON data or data is moved"
+            };
+          }
+        } catch (e) {
+          console.log(e);
+          throw { message: "Can't find ad data JSON" };
+        }
+      }
+    } catch (e) {
+      console.error("[PROSTOR] Exception caught:", e.message);
+      return [];
+    }
+  }
+
+  static transformRealEstateData(realEstateData) {
+    try {
+      const { lat, lng, property_name, price, size, link } = realEstateData;
+
+      // link contains part of the URL in the format of : /prodaja/stan/stup/9556
+      // general form is : /actionType/realEstateType/location/realEstateID
+      // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
+
+      const linkParts = link.split("/");
+
+      const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
+      const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
+      const prostorId = linkParts[4];
+      const url = `https://prostor.ba${link}`;
+
+      if (!adType || !realEstateType || !prostorId) {
+        return null;
+      }
+
+      const adStatus = AD_STATUS.STATUS_NORMAL;
+      const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
+      const parsedArea = parseFloat(size);
+
+      const data = {
+        url,
+        agencyObjectId: prostorId,
+        originAgencyName: AD_AGENCY.PROSTOR,
+        realEstateType,
+        adType,
+        title: property_name,
+        price: parsedPrice,
+        area: parsedArea,
+        gardenSize: null,
+        shortDescription: "",
+        longDescription: "",
+        streetNumber: 0,
+        streetName: "",
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat: lat,
+        locationLong: lng,
+        adStatus,
+        publishedDate: null,
+        renewedDate: null
+      };
+
+      return data;
+    } catch (e) {
+      console.error(
+        "[PROSTOR] Exception caught: " + e.message,
+        "\r\nURL:",
+        url
+      );
+      return null;
+    }
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  static getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "stan":
+        return AD_CATEGORY.FLAT.id;
+      case "kuca":
+        return AD_CATEGORY.HOUSE.id;
+      case "apartman":
+        return AD_CATEGORY.APARTMENT.id;
+      case "poslovni-prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "garaza":
+        return AD_CATEGORY.GARAGE.id;
+      case "zemljiste":
+        return AD_CATEGORY.LAND.id;
+      default:
+        return undefined;
+    }
+  }
+
+  static getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "prodaja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "najam":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = ProstorCrawler;

app/crawler/specificCrawlers/rental.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const RENTAL_ENUMS = {
+  RENTAL_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  RENTAL_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental");
+
+class RentalCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !RENTAL_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[RENTAL] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".pull-right")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[RENTAL] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    console.log("[RENTAL] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        const rentalId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromRentalId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromRentalActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const adStatus = AD_STATUS.STATUS_NORMAL;
+
+        const data = {
+          url,
+          agencyObjectId: rentalId,
+          originAgencyName: AD_AGENCY.RENTAL,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString()
+        };
+
+        return data;
+      } else {
+        console.log("[RENTAL] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromRentalId(rentalCategoryId) {
+    switch (rentalCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromRentalActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = RentalCrawler;

app/helpers/db/realEstate.js

@@ -0,0 +1,111 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+
+const bulkUpsertRealEstates = async realEstateData => {
+  try {
+    const fieldsToUpdateIfDuplicate = [
+      "realEstateType",
+      "adType",
+      "price",
+      "area",
+      "streetNumber",
+      "streetName",
+      "locality",
+      "municipality",
+      "city",
+      "region",
+      "entity",
+      "country",
+      "locationLat",
+      "locationLong",
+      "title",
+      "shortDescription",
+      "longDescription",
+      "gardenSize",
+      "adStatus",
+      "updatedAt",
+      "renewedDate"
+    ];
+    const order = [["updatedAt", "desc"]];
+
+    return await db.RealEstate.bulkCreate(realEstateData, {
+      updateOnDuplicate: fieldsToUpdateIfDuplicate,
+      returning: true,
+      order
+    });
+  } catch (e) {
+    console.log("Error bulk upserting realEstates : ", e);
+  }
+};
+
+const getRealEstateById = async id => {
+  return db.RealEstate.findByPk(id);
+};
+
+const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
+  const {
+    priceMin,
+    priceMax,
+    sizeMin,
+    sizeMax,
+    adType,
+    realEstateType,
+    areaToSearch
+  } = searchRequest;
+
+  const longitudeColumn = sequelize.col("locationLong");
+  const latitudeColumn = sequelize.col("locationLat");
+
+  const pointGeometry = sequelize.fn(
+    "ST_Point",
+    longitudeColumn,
+    latitudeColumn
+  );
+  const pointWithSRID = sequelize.fn("ST_SetSRID", pointGeometry, 4326);
+  const areaToSearchAsGeometry = sequelize.fn(
+    "ST_GeomFromGeoJSON",
+    JSON.stringify(areaToSearch)
+  );
+  const areaToSearchWithSRID = sequelize.fn(
+    "ST_SetSRID",
+    areaToSearchAsGeometry,
+    4326
+  );
+  const contains = sequelize.fn(
+    "ST_Contains",
+    areaToSearchWithSRID,
+    pointWithSRID
+  );
+
+  const geoSearchQueryPart = sequelize.where(contains, true);
+
+  const query = {
+    adType,
+    realEstateType,
+    price: {
+      [Op.lte]: priceMax,
+      [Op.gte]: priceMin
+    },
+    area: {
+      [Op.lte]: sizeMax,
+      [Op.gte]: sizeMin
+    },
+    [Op.and]: geoSearchQueryPart
+  };
+
+  const order = [["updatedAt", "desc"]];
+
+  return await db.RealEstate.findAll({
+    where: query,
+    limit: maxResults,
+    order
+  });
+};
+
+module.exports = {
+  bulkUpsertRealEstates,
+  getRealEstateById,
+  findRealEstatesForSearchRequest
+};

app/helpers/db/searchRequest.js

@@ -0,0 +1,74 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+
+const getSearchRequest = async searchRequestId => {
+  try {
+    return await db.SearchRequest.findByPk(searchRequestId);
+  } catch (error) {
+    return null;
+  }
+};
+
+const createSearchRequest = async (searchRequestFields = {}) => {
+  return await db.SearchRequest.create(searchRequestFields);
+};
+
+const findSearchRequestsForRealEstate = async realEstate => {
+  const {
+    price,
+    area,
+    adType,
+    realEstateType,
+    locationLat,
+    locationLong
+  } = realEstate;
+
+  if (!locationLat || !locationLong) {
+    return [];
+  }
+
+  const stGeometry = sequelize.fn(
+    "ST_GEOMFROMTEXT",
+    `POINT (${locationLong} ${locationLat})`,
+    4326
+  );
+  const areaToSearchColumn = sequelize.col("areaToSearch");
+  const contains = sequelize.fn("ST_Contains", areaToSearchColumn, stGeometry);
+
+  const geoSearchQueryPart = sequelize.where(contains, true);
+
+  const query = {
+    adType,
+    realEstateType,
+    subscribed: true,
+    [Op.and]: geoSearchQueryPart
+  };
+
+  if (price) {
+    query.priceMin = {
+      [Op.lte]: price
+    };
+    query.priceMax = {
+      [Op.gte]: price
+    };
+  }
+
+  if (area) {
+    query.sizeMin = {
+      [Op.lte]: area
+    };
+    query.sizeMax = {
+      [Op.gte]: area
+    };
+  }
+
+  return await db.SearchRequest.findAll({ where: query });
+};
+
+module.exports = {
+  getSearchRequest,
+  createSearchRequest,
+  findSearchRequestsForRealEstate
+};

app/helpers/db/searchRequestMatch.js

@@ -0,0 +1,36 @@
+"use strict";
+const db = require("../../models/index");
+
+const findRealEstatesForSearchRequest = async searchRequestId => {
+  const query = {
+    searchRequestId
+  };
+
+  const realEstatesModel = { model: db.RealEstate, as: "realEstates" };
+  const order = [[realEstatesModel, "updatedAt", "desc"]];
+  const include = [realEstatesModel];
+
+  const matches = await db.SearchRequestMatch.findAll({
+    where: query,
+    include,
+    order
+  });
+
+  const matchingRealEstates = [];
+  for (const match of matches) {
+    matchingRealEstates.push(...match.realEstates);
+  }
+
+  return matchingRealEstates;
+};
+
+const addMatches = async matchingRecords => {
+  return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
+    ignoreDuplicates: true
+  });
+};
+
+module.exports = {
+  findRealEstatesForSearchRequest,
+  addMatches
+};

app/helpers/email.js

@@ -0,0 +1,8 @@
+const isValidEmail = email => {
+  const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+  return email && email.length < 250 && simpleEmailRegex.test(email);
+};
+
+module.exports = {
+  isValidEmail
+};

app/helpers/emailContentGenerator.js

@@ -0,0 +1,112 @@
+"use strict";
+
+const { MAX_REAL_ESTATES_IN_EMAIL, APP_URL } = require("../config/appConfig");
+const { AD_CATEGORY } = require("../common/enums");
+
+const generateEmailFooter = searchRequestId => {
+  return `<div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
+     <div>Ako želite pogledati ili promijeniti uslove za ovu pretragu, <a href="${APP_URL}/pregled/${searchRequestId}">pogledajte ovdje</a></div>
+     <br/> 
+     <strong>Vaš,<br/>Kivi tim</strong>`;
+};
+
+const generateRealEstateLinks = realEstates => {
+  let realEstateLinks = "";
+  for (const realEstate of realEstates) {
+    const { id: realEstateId, title } = realEstate;
+
+    realEstateLinks += `<li><a href="${APP_URL}/redirect/${realEstateId}">${title}</a></li>`;
+  }
+  return realEstateLinks;
+};
+
+const generateNotificationEmail = (realEstates, searchRequestId) => {
+  const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
+  const realEstatesToShow = truncateList
+    ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
+    : realEstates;
+
+  const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
+  const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
+  const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
+  const emailFooter = generateEmailFooter(searchRequestId);
+
+  return `<h3>Zdravo</h3>
+  <h4>Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi</h4>
+  <div>
+  ${realEstateLinks}
+  <div/>
+  ${moreRealEstates}
+  </div>
+  <br/>
+  ${emailFooter}`;
+};
+
+const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  const {
+    id,
+    gardenSizeMin,
+    gardenSizeMax,
+    sizeMin,
+    sizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest;
+
+  const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
+  const instantRealEstatesText = `<br/>
+       <div>
+        U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
+        ${realEstateLinks}
+       </div>`;
+
+  const gardenSize = realEstateType.hasGardenSize
+    ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
+    : ``;
+
+  const emailFooter = generateEmailFooter(id);
+
+  return `<h3>Zdravo</h3>
+  <div>Naručili ste da Vam javimo ako se nekretnina sa navedenim uslovima pojavi u oglasima:</div>
+  <br/>
+  <div>
+    <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
+    ${gardenSize}
+    <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
+  </div>
+  ${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
+  <br/>
+  ${emailFooter}`;
+};
+
+const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
+  if (numberOfRealEstates === 1) {
+    return `Kivi: ${singleRealEstateTitle}`;
+  }
+
+  const leastSignificantDigit = numberOfRealEstates % 10;
+  const numberWithoutLastDigit = Math.floor(numberOfRealEstates / 10);
+  const secondLeastSignificantDigit = numberWithoutLastDigit % 10;
+
+  if (leastSignificantDigit === 1 && secondLeastSignificantDigit !== 1) {
+    return `Kivi : ${numberOfRealEstates} nova nekretnina`;
+  }
+
+  if (
+    leastSignificantDigit >= 2 &&
+    leastSignificantDigit <= 4 &&
+    secondLeastSignificantDigit !== 1
+  ) {
+    return `Kivi: ${numberOfRealEstates} nove nekretnine`;
+  }
+
+  return `Kivi: ${numberOfRealEstates} novih nekretnina`;
+};
+
+module.exports = {
+  generateNotificationEmail,
+  generateNewSearchRequestEmail,
+  generateEmailSubject
+};

app/helpers/forceSSL.js

@@ -0,0 +1,25 @@
+/**
+ * Force load with https on production environment
+ * https://devcenter.heroku.com/articles/http-routing#heroku-headers
+ */
+module.exports = function(environments, status) {
+  environments = environments || ["production"];
+  status = status || 301;
+  // console.log("New force SSL ");
+  // console.log("\tenvs : ", environments);
+  // console.log("\tstatus: ", status);
+  // console.log("\tENV : ", process.env.NODE_ENV);
+  return function(req, res, next) {
+    if (environments.indexOf(process.env.NODE_ENV) >= 0) {
+      if (req.headers["x-forwarded-proto"] !== "https") {
+        const urlToRedirectTo = `https://${req.hostname}${req.originalUrl}`;
+        // console.log("\tRedirect :", urlToRedirectTo);
+        res.redirect(status, urlToRedirectTo);
+      } else {
+        next();
+      }
+    } else {
+      next();
+    }
+  };
+};

app/helpers/url.js

@@ -1,13 +1,12 @@
-const db = require('../models/index');
+const { getSearchRequest } = require("./db/searchRequest");
 
-const currentRERequest = async (req) => {
-  const uniqueId = req.params['request_id'];
-  if(!uniqueId) return null;
-
-  const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
-  return request;
-}
+const currentSearchRequest = async req => {
+  const searchRequestId =
+    req && req.params ? req.params["searchRequestId"] : null;
+  if (!searchRequestId) return null;
 
+  return await getSearchRequest(searchRequestId);
+};
 module.exports = {
-  currentRERequest
-}
+  currentSearchRequest
+};

app/lib/arethereanynewitems.js

@@ -1,8 +0,0 @@
-const convertToDate = require("./convertToDate");
-function areThereAnyNewItems(lastItemDate, controlDate) {
-  if (!lastItemDate) {
-    return true;
-  }
-  return new Date(controlDate) < convertToDate(lastItemDate);
-}
-module.exports = areThereAnyNewItems;

app/lib/convertToDate.js

@@ -1,13 +0,0 @@
-function convertToDate(date) {
-  const [dan, mjesec, godina] = date
-    .split(". u ")[0]
-    .split(".")
-    .map(el => Number(el));
-  const [sati, minute] = date
-    .split(". u ")[1]
-    .split(":")
-    .map(el => Number(el));
-  return new Date(godina, mjesec, dan, sati, minute);
-}
-
-module.exports = convertToDate;

app/lib/scraptheitems.js

@@ -1,42 +0,0 @@
-let fetch = require("node-fetch");
-let cheerio = require("cheerio");
-const areThereAnyNewItems = require("./arethereanynewitems");
-
-async function scrapTheItems(url, controlDate, noNewItems = false) {
-  let items = [];
-  let response = await fetch(url);
-  const body = await response.text();
-  const $ = cheerio.load(body);
-  $("#rezultatipretrage")
-    .find(".listitem")
-    .each(async (index, elem) => {
-      if (noNewItems) return;
-      const itemDate = $(elem)
-        .find(".cijena > .datum > div")
-        .first()
-        .attr("data-cijelidatum");
-
-      if (controlDate && !areThereAnyNewItems(itemDate, controlDate)) {
-        noNewItems = true;
-        return;
-      }
-
-      const id = $(elem)
-        .find("a")
-        .first()
-        .attr("href");
-      const cijena = $(elem)
-        .find(".cijena > .datum > span")
-        .first()
-        .text();
-      const image = $(elem)
-        .find("a > .slika > img")
-        .first()
-        .attr("src");
-
-      items.push({ url: id, price: cijena, image, date: itemDate });
-    });
-  return items;
-}
-
-module.exports = scrapTheItems;

app/lib/sendnotification.js

@@ -1,58 +0,0 @@
-const scrapTheItems = require("./scraptheitems");
-const convertToDate = require("./convertToDate");
-const AWS = require('aws-sdk');
-AWS.config.update({region: 'eu-central-1'});
-
-
-async function sendNotification(marketAlert) {
-  const { id, email, olx_url, last_date } = marketAlert;
-  let url =
-    "https://www.olx.ba/pretraga?" + olx_url + "&sort_order=desc&sort_po=datum";
-  let newItems = await scrapTheItems(url);
-  let lastDate = newItems.length && newItems[0].date;
-  let message =
-    newItems.length &&
-    newItems.reduce(
-      (mes, item) => mes + `<strong>${item.url} i  ${item.price}</strong>`,
-      ""
-    );
-
-  // Create sendEmail params 
-  var params = {
-    Destination: { /* required */
-      CcAddresses: [
-      ],
-      ToAddresses: [
-        email
-      ]
-    },
-    Message: { /* required */
-      Body: { /* required */
-        Html: {
-         Charset: "UTF-8",
-         Data: message
-        },
-        Text: {
-         Charset: "UTF-8",
-         Data: message // TODO: convert to text
-        }
-       },
-     Subject: {
-      Charset: 'UTF-8',
-      Data: 'Javimi alert'
-     }
-    },
-    Source: 'info@saburly.com', /* required */
-    ReplyToAddresses: [
-       'info@saburly.com',
-    ],
-  };
-
-  if (message) {
-    const sendPromise = new AWS.SES({apiVersion: '2010-12-01'}).sendEmail(params).promise();
-    await sendPromise; 
-    return { id, date: String(convertToDate(lastDate)) };
-  }
-}
-
-module.exports = sendNotification;

app/migrations/20190417035319-create-market-alert.js

@@ -1,7 +1,7 @@
-'use strict';
+"use strict";
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('MarketAlerts', {
+    return queryInterface.createTable("MarketAlerts", {
       id: {
         allowNull: false,
         autoIncrement: true,
@@ -29,6 +29,6 @@ module.exports = {
     });
   },
   down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('MarketAlerts');
+    return queryInterface.dropTable("MarketAlerts");
   }
 };

app/migrations/20190417035707-create-real-estate-request.js

@@ -1,7 +1,7 @@
-'use strict';
+"use strict";
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('RealEstateRequests', {
+    return queryInterface.createTable("RealEstateRequests", {
       id: {
         allowNull: false,
         autoIncrement: true,
@@ -12,8 +12,7 @@ module.exports = {
         type: Sequelize.UUID
       },
       realEstateType: {
-        type: Sequelize.ENUM,  
-        values: ['kuca','stan','vikendica','plac','poslovni_prostor','apartman','garaza']
+        type: Sequelize.STRING
       },
       email: {
         type: Sequelize.STRING
@@ -29,6 +28,6 @@ module.exports = {
     });
   },
   down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('RealEstateRequests');
+    return queryInterface.dropTable("RealEstateRequests");
   }
 };

app/migrations/20190427043621-add_city_to_real_estate_request.js

@@ -1,18 +1,15 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.addColumn(
-      'RealEstateRequests',
-      'city',
-     Sequelize.STRING
+      "RealEstateRequests",
+      "city",
+      Sequelize.STRING
     );
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-      'RealEstateRequests',
-      'city'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "city");
   }
 };

app/migrations/20190429050222-add_place_to_real_estate_request.js

@@ -1,18 +1,15 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.addColumn(
-      'RealEstateRequests',
-      'place',
-     Sequelize.STRING
+      "RealEstateRequests",
+      "place",
+      Sequelize.STRING
     );
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-      'RealEstateRequests',
-      'place'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "place");
   }
 };

app/migrations/20190516180226-rename-place-column.js

@@ -0,0 +1,19 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "place",
+      "municipality"
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "municipality",
+      "place"
+    );
+  }
+};

app/migrations/20190516222240-rename-city-column.js

@@ -0,0 +1,11 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn("RealEstateRequests", "city", "region");
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn("RealEstateRequests", "region", "city");
+  }
+};

app/migrations/20190517072957-add-size-to-real-estate-request.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstateRequests", "size", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "size");
+  }
+};

app/migrations/20190517090015-add-gardenSize-to-real-estate-request.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstateRequests", "gardenSize", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "gardenSize");
+  }
+};

app/migrations/20190517092716-add-price-to-real-estate-request.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstateRequests", "price", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "price");
+  }
+};

app/migrations/20190523144812-activate-postgis.js

@@ -0,0 +1,19 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query("CREATE EXTENSION postgis")
+      .then(([results, metadata]) => {
+        /// No result
+      });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query("DROP EXTENSION IF EXISTS postgis")
+      .then(([results, metadata]) => {
+        /// No result
+      });
+  }
+};

app/migrations/20190523151420-add-bounding-box-column.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query(
+        'ALTER TABLE "RealEstateRequests" ADD COLUMN bounding_box geometry(Polygon);'
+      )
+      .then(([results, metadata]) => {
+        /// No result
+      });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query('ALTER TABLE "RealEstateRequests" DROP COLUMN bounding_box')
+      .then(([results, metadata]) => {
+        /// No result
+      });
+  }
+};

app/migrations/20190529093410-slider-fields.js

@@ -0,0 +1,48 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceRange", {
+          transaction: t
+        })
+      ]);
+    });
+  }
+};

app/migrations/20190530101945-range-fields.js

@@ -0,0 +1,147 @@
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "size", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSize", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "price", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeMax", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "sizeMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "sizeMax", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceMin", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMax",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "size",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSize",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "price",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  }
+};

app/migrations/20190531111232-subscribed-boolean.js

@@ -0,0 +1,15 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn(
+      "RealEstateRequests",
+      "subscribed",
+      Sequelize.BOOLEAN
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "subscribed");
+  }
+};

app/migrations/20190618103020-expand-maketalert.js

@@ -0,0 +1,70 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "size",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "gardenSize",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "price",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "municipality",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "region",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "size", { transaction: t }),
+        queryInterface.removeColumn("MarketAlerts", "gardenSize", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "price", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "municipality", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "region", {
+          transaction: t
+        })
+      ]);
+    });
+  }
+};

app/migrations/20190618124522-marketalerts-additional-info.js

@@ -0,0 +1,59 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "olxUrl", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "url",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "realestateOrigin",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "originId",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "url", { transaction: t }),
+        queryInterface.removeColumn("MarketAlerts", "realestateOrigin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "originId", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "olxUrl",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  }
+};

app/migrations/20190621162321-add-category-to-marketalert.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "realEstateType", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "realEstateType");
+  }
+};

app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "notified", {
+      type: Sequelize.BOOLEAN
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "notified");
+  }
+};

app/migrations/20190628165512-add-title-to-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "title", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "title");
+  }
+};

app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "request", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "request");
+  }
+};

app/migrations/20190710141356-add-has-location-to-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "hasLocation", {
+      type: Sequelize.BOOLEAN
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "hasLocation");
+  }
+};

app/migrations/20190909110623-add-location-input-to-real-estate-request.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstateRequests", "locationInput", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "locationInput");
+  }
+};

app/migrations/20190910033923-change-bounding-box-from-snake-case-to-camel-case.js

@@ -0,0 +1,19 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "bounding_box",
+      "boundingBox"
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "boundingBox",
+      "bounding_box"
+    );
+  }
+};

app/migrations/20190912191829-add-realEstates-table.js

@@ -0,0 +1,72 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      url: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      agencyObjectId: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      originAgencyName: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      realEstateType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      adType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      price: Sequelize.REAL,
+      area: Sequelize.REAL,
+      gardenSize: Sequelize.REAL,
+      streetNumber: Sequelize.INTEGER,
+      streetName: Sequelize.TEXT,
+      locality: Sequelize.TEXT,
+      municipality: Sequelize.TEXT,
+      city: Sequelize.TEXT,
+      region: Sequelize.TEXT,
+      entity: Sequelize.TEXT,
+      country: Sequelize.TEXT,
+      locationLat: Sequelize.REAL,
+      locationLong: Sequelize.REAL,
+      lastTimeCrawled: {
+        type: Sequelize.DATE,
+        allowNull: false
+      },
+      deleted: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false
+      },
+      sold: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("RealEstates", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("RealEstates", {});
+  }
+};

app/migrations/20190912215313-add-searchRequests-table.js

@@ -0,0 +1,79 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.UUID,
+        defaultValue: Sequelize.UUIDV4,
+        allowNull: false,
+        primaryKey: true
+      },
+      areaToSearch: {
+        type: Sequelize.GEOMETRY("POLYGON", 4326),
+        allowNull: false,
+        defaultValue: {
+          type: "Polygon",
+          coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+          crs: { type: "name", properties: { name: "EPSG:4326" } }
+        }
+      },
+      realEstateType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      adType: {
+        type: Sequelize.TEXT,
+        allowNull: false,
+        defaultValue: "sell"
+      },
+      email: Sequelize.TEXT,
+      locality: Sequelize.TEXT,
+      municipality: Sequelize.TEXT,
+      city: Sequelize.TEXT,
+      region: Sequelize.TEXT,
+      entity: Sequelize.TEXT,
+      country: Sequelize.TEXT,
+      sizeMin: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      sizeMax: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      priceMin: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      priceMax: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      gardenSizeMin: Sequelize.INTEGER,
+      gardenSizeMax: Sequelize.INTEGER,
+      subscribed: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false,
+        defaultValue: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("SearchRequests", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("SearchRequests", {});
+  }
+};

app/migrations/20190912215556-add-searchRequestMatches-table.js

@@ -0,0 +1,53 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false
+      },
+      searchRequestId: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "SearchRequests",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      notified: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false,
+        defaultValue: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("SearchRequestMatches", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("SearchRequestMatches", {});
+  }
+};

app/migrations/20190916160603-add-title-column-to-realEstates.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "title", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "title");
+  }
+};

app/migrations/20190916230229-add-description-columns-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "shortDescription", {
+        type: Sequelize.STRING
+      }),
+      queryInterface.addColumn("RealEstates", "longDescription", {
+        type: Sequelize.STRING
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "shortDescription"),
+      queryInterface.removeColumn("RealEstates", "longDescription")
+    ]);
+  }
+};

app/migrations/20190916231241-add-status-column-to-realEstates.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "adStatus", {
+      type: Sequelize.INTEGER
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "adStatus");
+  }
+};

app/migrations/20190917090318-add-composite-key-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addConstraint(
+      "RealEstates",
+      ["originAgencyName", "agencyObjectId"],
+      {
+        type: "unique",
+        name: "agencyNameObjectIdUniqueKey"
+      }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeConstraint(
+      "RealEstates",
+      "agencyNameObjectIdUniqueKey"
+    );
+  }
+};

app/migrations/20190917101052-remove-lastTimeCrawled-column-from-realEstates.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "lastTimeCrawled");
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "lastTimeCrawled", {
+      type: Sequelize.DATE,
+      notNull: true
+    });
+  }
+};

app/migrations/20190917102437-remove-deleted-sold-columns-from-realEstates.js

@@ -0,0 +1,23 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "deleted"),
+      queryInterface.removeColumn("RealEstates", "sold")
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "deleted", {
+        type: Sequelize.BOOLEAN,
+        notNull: true
+      }),
+      queryInterface.addColumn("RealEstates", "sold", {
+        type: Sequelize.BOOLEAN,
+        notNull: true
+      })
+    ]);
+  }
+};

app/migrations/20190917115804-change-title-short-long-description-columns-in-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.changeColumn("RealEstates", "shortDescription", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.changeColumn("RealEstates", "longDescription", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.changeColumn("RealEstates", "title", {
+        type: Sequelize.TEXT
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([]);
+  }
+};

app/migrations/20190923185802-add-published-renewed-dates-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "publishedDate", {
+        type: Sequelize.DATE
+      }),
+      queryInterface.addColumn("RealEstates", "renewedDate", {
+        type: Sequelize.DATE
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "renewedDate"),
+      queryInterface.removeColumn("RealEstates", "publishedDate")
+    ]);
+  }
+};

app/migrations/20190927121449-change-adType-in-searchRequests-table-to-SALE.js

@@ -0,0 +1,15 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'SALE' WHERE "adType" = 'sell';`
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'sell' WHERE "adType" = 'SALE';`
+    );
+  }
+};

app/migrations/20190927180101-change-change-realEstateType-in-searchRequests-table.js

@@ -0,0 +1,31 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'HOUSE' WHERE "realEstateType" = 'kuca';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'FLAT' WHERE "realEstateType" = 'stan';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'COTTAGE' WHERE "realEstateType" = 'vikendica';`
+      )
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'kuca' WHERE "realEstateType" = 'HOUSE';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'stan' WHERE "realEstateType" = 'FLAT';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'vikendica' WHERE "realEstateType" = 'COTTAGE';`
+      )
+    ]);
+  }
+};

app/models/index.js

@@ -1,27 +1,39 @@
-'use strict';
+"use strict";
 
-const fs = require('fs');
-const path = require('path');
-const Sequelize = require('sequelize');
+const fs = require("fs");
+const path = require("path");
+const Sequelize = require("sequelize");
 const basename = path.basename(__filename);
-const env = process.env.NODE_ENV || 'development';
-const config = require(__dirname + '/../config/config.json')[env];
+const env = process.env.NODE_ENV || "development";
+const config = require(__dirname + "/../config/config.json")[env];
 const db = {};
 
+config.username = process.env.DB_USERNAME || config.username;
+config.password = process.env.DB_PASSWORD || config.password;
+config.database = process.env.DB_NAME || config.database;
+config.port = process.env.DB_PORT || config.port;
+config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;
+
 let sequelize;
 if (config.use_env_variable) {
   sequelize = new Sequelize(process.env[config.use_env_variable], config);
 } else {
-  sequelize = new Sequelize(config.database, config.username, config.password, config);
+  sequelize = new Sequelize(
+    config.database,
+    config.username,
+    config.password,
+    config
+  );
 }
 
-fs
-  .readdirSync(__dirname)
+fs.readdirSync(__dirname)
   .filter(file => {
-    return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
+    return (
+      file.indexOf(".") !== 0 && file !== basename && file.slice(-3) === ".js"
+    );
   })
   .forEach(file => {
-    const model = sequelize['import'](path.join(__dirname, file));
+    const model = sequelize["import"](path.join(__dirname, file));
     db[model.name] = model;
   });
 

app/models/marketalert.js

@@ -1,15 +0,0 @@
-'use strict';
-module.exports = (sequelize, DataTypes) => {
-  const MarketAlert = sequelize.define('MarketAlert', {
-    olxUrl: DataTypes.STRING,
-    lastDate: DataTypes.STRING,
-    email: { 
-      type: DataTypes.STRING,
-      allowNul: false
-    }
-  }, {});
-  MarketAlert.associate = function(models) {
-    // associations can be defined here
-  };
-  return MarketAlert;
-};

app/models/realEstate.js

@@ -0,0 +1,55 @@
+"use strict";
+
+module.exports = (sequelize, DataTypes) => {
+  const RealEstate = sequelize.define("RealEstate", {
+    id: {
+      type: DataTypes.BIGINT,
+      autoIncrement: true,
+      allowNull: false,
+      primaryKey: true
+    },
+    url: {
+      type: DataTypes.TEXT,
+      allowNull: false
+    },
+    originAgencyName: {
+      type: DataTypes.TEXT,
+      allowNull: false,
+      unique: true
+    },
+    agencyObjectId: {
+      type: DataTypes.TEXT,
+      allowNull: false,
+      unique: true
+    },
+    realEstateType: {
+      type: DataTypes.TEXT,
+      allowNull: false
+    },
+    adType: {
+      type: DataTypes.TEXT,
+      allowNull: false
+    },
+    price: DataTypes.REAL,
+    area: DataTypes.REAL,
+    gardenSize: DataTypes.REAL,
+    streetNumber: DataTypes.INTEGER,
+    streetName: DataTypes.TEXT,
+    locality: DataTypes.TEXT,
+    municipality: DataTypes.TEXT,
+    city: DataTypes.TEXT,
+    region: DataTypes.TEXT,
+    entity: DataTypes.TEXT,
+    country: DataTypes.TEXT,
+    locationLat: DataTypes.REAL,
+    locationLong: DataTypes.REAL,
+    title: DataTypes.TEXT,
+    shortDescription: DataTypes.TEXT,
+    longDescription: DataTypes.TEXT,
+    adStatus: DataTypes.INTEGER,
+    publishedDate: DataTypes.DATE,
+    renewedDate: DataTypes.DATE
+  });
+
+  return RealEstate;
+};

app/models/realestaterequest.js

@@ -1,21 +0,0 @@
-'use strict';
-module.exports = (sequelize, DataTypes) => {
-  const RealEstateRequest = sequelize.define('RealEstateRequest', {
-    uniqueId: {
-      type: DataTypes.UUID,
-      defaultValue: DataTypes.UUIDV4,
-      allowNull: false
-    },
-    realEstateType: {
-      type: DataTypes.ENUM,
-      values: ['kuca','stan','vikendica','plac','poslovni_prostor','apartman','garaza']
-    },
-    email: DataTypes.STRING, 
-    city: DataTypes.STRING,
-    place: DataTypes.STRING,
-  }, {});
-  RealEstateRequest.associate = function(models) {
-    // associations can be defined here
-  };
-  return RealEstateRequest;
-};

app/models/searchRequest.js

@@ -0,0 +1,68 @@
+"use strict";
+
+const { AD_TYPE } = require("../common/enums");
+
+module.exports = (sequelize, DataTypes) => {
+  const SearchRequest = sequelize.define("SearchRequest", {
+    id: {
+      type: DataTypes.UUID,
+      defaultValue: DataTypes.UUIDV4,
+      allowNull: false,
+      primaryKey: true
+    },
+    areaToSearch: {
+      type: DataTypes.GEOMETRY("POLYGON", 4326),
+      allowNull: false,
+      defaultValue: {
+        type: "Polygon",
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+        crs: { type: "name", properties: { name: "EPSG:4326" } }
+      }
+    },
+    realEstateType: {
+      type: DataTypes.TEXT,
+      allowNull: false
+    },
+    adType: {
+      type: DataTypes.TEXT,
+      allowNull: false,
+      defaultValue: AD_TYPE.AD_TYPE_SALE.stringId
+    },
+    email: DataTypes.TEXT,
+    locality: DataTypes.TEXT,
+    municipality: DataTypes.TEXT,
+    city: DataTypes.TEXT,
+    region: DataTypes.TEXT,
+    entity: DataTypes.TEXT,
+    country: DataTypes.TEXT,
+    sizeMin: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 0
+    },
+    sizeMax: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 0
+    },
+    priceMin: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 0
+    },
+    priceMax: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 0
+    },
+    gardenSizeMin: DataTypes.INTEGER,
+    gardenSizeMax: DataTypes.INTEGER,
+    subscribed: {
+      type: DataTypes.BOOLEAN,
+      defaultValue: false,
+      allowNull: false
+    }
+  });
+
+  return SearchRequest;
+};

app/models/searchRequestMatch.js

@@ -0,0 +1,54 @@
+"use strict";
+
+module.exports = (sequelize, DataTypes) => {
+  const SearchRequestMatch = sequelize.define(
+    "SearchRequestMatch",
+    {
+      id: {
+        type: DataTypes.BIGINT,
+        autoIncrement: true,
+        allowNull: false
+      },
+      realEstateId: {
+        type: DataTypes.BIGINT,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "RealEstate",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      searchRequestId: {
+        type: DataTypes.UUID,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "SearchRequest",
+          key: "id"
+        }
+      },
+      notified: {
+        type: DataTypes.BOOLEAN,
+        allowNull: false,
+        defaultValue: false
+      }
+    },
+    {
+      name: {
+        singular: "searchRequestMatch",
+        plural: "searchRequestMatches"
+      }
+    }
+  );
+
+  SearchRequestMatch.associate = models => {
+    SearchRequestMatch.hasMany(models.RealEstate, {
+      foreignKey: "id",
+      as: "realEstates"
+    });
+  };
+
+  return SearchRequestMatch;
+};

app/oldModels/marketalert.js

@@ -0,0 +1,32 @@
+"use strict";
+module.exports = (sequelize, DataTypes) => {
+  const MarketAlert = sequelize.define(
+    "MarketAlert",
+    {
+      url: DataTypes.STRING,
+      realestateOrigin: DataTypes.STRING,
+      originId: DataTypes.STRING,
+      lastDate: DataTypes.STRING,
+      size: DataTypes.INTEGER,
+      gardenSize: DataTypes.INTEGER,
+      price: DataTypes.INTEGER,
+      municipality: DataTypes.STRING,
+      region: DataTypes.STRING,
+      realEstateType: DataTypes.STRING,
+      notified: DataTypes.BOOLEAN,
+      title: DataTypes.STRING,
+      request: DataTypes.STRING,
+      hasLocation: DataTypes.BOOLEAN,
+
+      email: {
+        type: DataTypes.STRING,
+        allowNul: false
+      }
+    },
+    {}
+  );
+  MarketAlert.associate = function(models) {
+    // associations can be defined here
+  };
+  return MarketAlert;
+};

app/oldModels/realestaterequest.js

@@ -0,0 +1,32 @@
+"use strict";
+
+module.exports = (sequelize, DataTypes) => {
+  const RealEstateRequest = sequelize.define(
+    "RealEstateRequest",
+    {
+      uniqueId: {
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4,
+        allowNull: false
+      },
+      realEstateType: DataTypes.STRING,
+      email: DataTypes.STRING,
+      region: DataTypes.STRING,
+      municipality: DataTypes.STRING,
+      sizeMin: DataTypes.INTEGER,
+      sizeMax: DataTypes.INTEGER,
+      gardenSizeMin: DataTypes.INTEGER,
+      gardenSizeMax: DataTypes.INTEGER,
+      priceMin: DataTypes.INTEGER,
+      priceMax: DataTypes.INTEGER,
+      boundingBox: DataTypes.GEOMETRY("POINT", 4326),
+      subscribed: DataTypes.BOOLEAN,
+      locationInput: DataTypes.STRING
+    },
+    {}
+  );
+  RealEstateRequest.associate = function(models) {
+    // associations can be defined here
+  };
+  return RealEstateRequest;
+};

app/public/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/mstile-150x150.png"/>
+            <TileColor>#da532c</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

app/public/images/logo.svg

@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="61"
+   height="94"
+   viewBox="0 0 61 94"
+   version="1.1"
+   id="svg29"
+   sodipodi:docname="logo2.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata33">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1853"
+     inkscape:window-height="1025"
+     id="namedview31"
+     showgrid="false"
+     inkscape:zoom="5.6568543"
+     inkscape:cx="29.58234"
+     inkscape:cy="42.092869"
+     inkscape:window-x="67"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg29" />
+  <!-- Generator: Sketch 51.3 (57544) - http://www.bohemiancoding.com/sketch -->
+  <desc
+     id="desc2">Created with Sketch.</desc>
+  <defs
+     id="defs4" />
+  <g
+     id="g938"
+     transform="translate(4.6566166)">
+    <g
+       id="Group"
+       transform="translate(21.468225,75.05246)"
+       style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
+      <path
+         style="fill:#02adba;fill-opacity:1"
+         d="m 2.8937203,1.377828 h 0.00886 V 16.18948 h -0.00886 c -0.080215,0.775019 -0.6954121,1.377828 -1.4424314,1.377828 -0.74701929,0 -1.36221635,-0.602809 -1.4424314,-1.377828 H 0 V 1.377828 H 0.0088575 C 0.08907255,0.60280831 0.70426961,0 1.4512889,0 2.1983082,0 2.8135053,0.60280831 2.8937203,1.377828 Z"
+         id="Combined-Shape"
+         inkscape:connector-curvature="0" />
+    </g>
+    <g
+       id="Group-Copy"
+       transform="translate(40.284746,75.05246)"
+       style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
+      <path
+         style="fill:#02adba;fill-opacity:1"
+         d="m 2.8937203,1.377828 h 0.00886 V 16.18948 h -0.00886 c -0.080215,0.775019 -0.6954121,1.377828 -1.4424314,1.377828 -0.74701929,0 -1.36221635,-0.602809 -1.4424314,-1.377828 H 0 V 1.377828 H 0.0088575 C 0.08907255,0.60280831 0.70426961,0 1.4512889,0 2.1983082,0 2.8135053,0.60280831 2.8937203,1.377828 Z"
+         id="path8"
+         inkscape:connector-curvature="0" />
+    </g>
+    <g
+       id="Group-2"
+       transform="translate(26.045022,75.05246)"
+       style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1">
+      <path
+         style="fill:#02adba;fill-opacity:1"
+         sodipodi:nodetypes="cccsscccsscccsc"
+         d="M 4.7335014,16.043249 0.01556995,2.0709177 v 0 C -0.03604093,1.9151618 -0.08984375,1.7196494 -0.08984375,1.5461504 -0.08984375,0.69007771 0.68116079,0 1.5214152,0 2.3524826,0 2.7388538,0.67107889 3.0425868,1.5220354 L 6.255723,11.044343 9.4667171,1.5292951 C 9.7627803,0.71794799 10.154547,0 10.987999,0 c 0.840254,0 1.603446,0.68226521 1.603446,1.5383379 0,0.1793218 -0.04867,0.3879772 -0.103645,0.5481862 v 0 L 7.776097,16.026251 c -0.6976248,1.816338 -0.6840821,1.541057 -1.5213901,1.541057 -0.831732,0 -0.9926068,0.05035 -1.5212055,-1.524059 z"
+         id="path11"
+         inkscape:connector-curvature="0" />
+    </g>
+    <path
+       sodipodi:nodetypes="ccccsscccsscccccscscccssscc"
+       inkscape:connector-curvature="0"
+       id="path14"
+       d="m 12.013315,76.606127 v 4.050014 l 4.717498,-5.019565 v 0 c 0.404985,-0.305017 0.852764,-0.412505 1.290558,-0.412505 0.899311,0 1.590893,0.796979 1.590893,1.653051 0,0.589329 -0.373586,0.895853 -0.854196,1.364015 l -5.796302,6.120383 5.715392,5.649676 c 0.532765,0.486219 0.935106,0.782903 0.935106,1.402972 0,0.856073 -0.729035,1.550057 -1.628346,1.550057 -0.420934,0 -0.861077,0.0072 -1.216312,-0.275429 v 0 l -4.754291,-4.690453 v 3.249235 h -0.0031 c 0.0021,0.03199 0.0031,0.06425 0.0031,0.09674 0,0.856072 -0.729035,1.550056 -1.628346,1.550056 -0.8993109,-10e-7 -1.6283459,-0.693985 -1.6283459,-1.550057 0,-0.03249 10e-4,-0.06475 0.0031,-0.09674 h -0.0031 v -14.64145 -0.0036 c 0,-0.856083 0.729035,-1.550067 1.6283459,-1.550067 0.899311,0 1.628346,0.693984 1.628346,1.550057 0,0.0012 -1e-6,0.0024 -4e-6,0.0036 z"
+       style="fill:#02adba;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:1" />
+  </g>
+  <g
+     style="fill:#02adba;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1"
+     id="Group-5-Copy"
+     transform="translate(0.30033447)">
+    <path
+       inkscape:connector-curvature="0"
+       id="Page-1-Copy-3"
+       d="m 28.864505,0.71910556 c 0.913861,-0.49697651 2.013643,-0.49697651 2.927503,0 L 58.602992,15.299478 c 1.00074,0.544224 1.624856,1.599801 1.624856,2.748147 V 52.95707 c 0,1.148497 -0.624278,2.204188 -1.625223,2.748347 L 31.791641,70.28107 c -0.913667,0.49671 -2.013102,0.49671 -2.926768,0 L 2.053889,55.705417 C 1.052944,55.161258 0.42866553,54.105567 0.42866553,52.95707 V 18.047625 c 0,-1.148346 0.62411557,-2.203923 1.62485607,-2.748147 z M 30.328257,3.4672523 3.5172731,18.047625 V 52.95707 L 30.328257,67.532724 57.13924,52.95707 V 18.047625 Z"
+       style="fill:#02adba;fill-opacity:1" />
+    <path
+       inkscape:connector-curvature="0"
+       transform="matrix(-1,0,0,1,38.607594,0)"
+       id="Page-1-Copy-2"
+       d="m 6.6881981,8.5296646 c 0,-1.1837877 1.2534136,-1.9364737 2.2824613,-1.3706383 L 31.113251,19.334421 c 0.496869,0.273211 0.806146,0.799054 0.806146,1.370639 v 29.035897 c 0,0.571661 -0.309358,1.097561 -0.806331,1.37074 L 8.970475,63.283151 C 7.9414326,63.848801 6.6881981,63.096106 6.6881981,61.912412 Z M 9.7768057,11.155344 V 59.287138 L 28.830789,48.813446 V 21.632428 Z"
+       style="fill:#02adba;fill-opacity:1" />
+    <path
+       inkscape:connector-curvature="0"
+       transform="rotate(4,36.513012,12.39682)"
+       id="Path-85"
+       d="m 29.442604,10.590184 12.454251,6.520214 c 0.879716,0.424975 1.970417,0.118743 2.436149,-0.683987 0.465733,-0.802729 0.130132,-1.797981 -0.749584,-2.222955 L 31.12917,7.6832415 C 30.249453,7.2582668 29.158752,7.564498 28.69302,8.367228 c -0.465732,0.8027299 -0.130132,1.797981 0.749584,2.222956 z"
+       style="fill:#02adba;fill-opacity:1" />
+  </g>
+  <path
+     style="fill:#02adba;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1"
+     inkscape:connector-curvature="0"
+     id="Path-87"
+     d="m 7.8821311,47.17753 c -0.7181691,0.486659 -0.8975584,1.451387 -0.4006772,2.15478 0.4968812,0.703394 1.4818741,0.879093 2.2000432,0.392434 L 31.518752,34.926994 c 0.939679,-0.636761 0.900745,-2.009481 -0.07358,-2.594168 L 9.6079198,19.228381 c -0.744646,-0.446859 -1.718161,-0.217874 -2.1744066,0.511452 -0.4562456,0.729326 -0.2224509,1.682812 0.5221952,2.129671 L 27.723653,33.732165 Z" />
+</svg>