Merge branch 'master' after user-agent change into email-density

Create fetch wrapper with user agent

See merge request saburly/marketalarm/web!98

.gitignore

@@ -1,2 +1,5 @@
 node_modules/
 .env
+.idea/
+.eslintrc
+.vscode/

.prettierignore

@@ -0,0 +1 @@
+*.ejs

Dockerfile

@@ -3,6 +3,7 @@ FROM postgres:11.3
 ENV POSTGIS_MAJOR 2.4
 
 RUN apt-get update \
+      && apt-get --assume-yes install postgresql-11-postgis-2.5-scripts\
       && apt-get --assume-yes install software-properties-common postgis\
       && rm -rf /var/lib/apt/lists/
 

README.md

@@ -4,6 +4,8 @@ The purpose of this project is to build a web application that enables subscribi
 
 ## Setup
 
+* Before setup please confirm that Docker is installed `docker --version`. If not install it from official site. 
+
 ### Setup with npm commands
 
 1. Install packages  
@@ -24,7 +26,7 @@ this will create and run postgres image and then execute migrations
 `docker build -t marketalerts .`
 
 2. Run postgres image with  
-`docker run --name pg_marketalerts -d -p 5432:5432 marketalerts`
+`docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts`
 
 3. Install packages  
 `npm install`
@@ -34,3 +36,11 @@ this will create and run postgres image and then execute migrations
 
 5. Run app  
 `npm start` or `npm run start-mon` to run app with automatic restart on code change
+
+
+### AWS SES
+
+    - AWS SES credentials are handled with env vratiables
+    - Notification emails are sent in batches of 50, by using SES templates
+    - Make sure that you are using different templates for different envirorments
+

app/common/enums.js

@@ -0,0 +1,422 @@
+const PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 85000],
+  range: {
+    min: [0],
+    max: [300000]
+  },
+  step: 1000,
+  connect: true
+};
+const FLAT_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [800000]
+  },
+  step: 5000,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [1500000]
+  },
+  step: 10000,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [15000, 50000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 2000,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [40000, 80000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 10000,
+  connect: true
+};
+const PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 500],
+  range: {
+    min: [0],
+    max: [2000]
+  },
+  step: 50,
+  connect: true
+};
+const FLAT_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 600],
+  range: {
+    min: [0],
+    max: [4000]
+  },
+  step: 100,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [10000]
+  },
+  step: 100,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [200, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
+//This will be used for Flats, Apartments, Houses
+const HOME_SIZE_SLIDER_OPTIONS = {
+  start: [30, 75],
+  range: {
+    min: [0],
+    max: [500]
+  },
+  step: 5,
+  connect: true
+};
+
+const OFFICE_SIZE_SLIDER_OPTIONS = {
+  start: [30, 150],
+  range: {
+    min: [0],
+    max: [1200]
+  },
+  step: 10,
+  connect: true
+};
+const GARDEN_SIZE_SLIDER_OPTIONS = {
+  start: [100, 1000],
+  range: {
+    min: [0],
+    max: [10000]
+  },
+  step: 100,
+  connect: true
+};
+
+const LAND_SIZE_SLIDER_OPTIONS = {
+  start: [5000, 15000],
+  range: {
+    min: [0],
+    max: [100000]
+  },
+  step: 100,
+  connect: true
+};
+const GARAGE_SIZE_SLIDER_OPTIONS = {
+  start: [10, 20],
+  range: {
+    min: [0],
+    max: [150]
+  },
+  step: 2,
+  connect: true
+};
+
+const GARAGE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [2000, 10000],
+  range: {
+    min: [0],
+    max: [60000]
+  },
+  step: 200,
+  connect: true
+};
+
+const GARAGE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [50, 150],
+  range: {
+    min: [0],
+    max: [1000]
+  },
+  step: 10,
+  connect: true
+};
+
+const AD_TYPE = {
+  AD_TYPE_SALE: {
+    id: 1,
+    stringId: "SALE",
+    title: "Kupi"
+  },
+  AD_TYPE_RENT: {
+    id: 2,
+    stringId: "RENT",
+    title: "Unajmi"
+  },
+  AD_TYPE_REQUEST: {
+    id: 3,
+    stringId: "REQUEST",
+    title: "Potražnja"
+  }
+};
+
+const AD_CATEGORY = {
+  ALL: {
+    id: "ALL"
+  },
+  FLAT: {
+    id: "FLAT",
+    title: "Stan",
+    hasGardenSize: false,
+    hasAccesRoadType: true,
+    hasBalconyProp: true,
+    hasNewBuildingProp: true,
+    hasElevatorProp: true,
+    hasNumberOfRoom: true,
+    hasNumberOfFloors: false,
+    hasFloorProp: true,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+  },
+  HOUSE: {
+    id: "HOUSE",
+    title: "Kuća",
+    hasGardenSize: true,
+    hasAccesRoadType: true,
+    hasBalconyProp: true,
+    hasNewBuildingProp: true,
+    hasElevatorProp: false,
+    hasNumberOfRoom: true,
+    hasNumberOfFloors: true,
+    hasFloorProp: false,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
+    gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
+  },
+  OFFICE: {
+    id: "OFFICE",
+    title: "Kancelarija",
+    hasGardenSize: false,
+    hasAccesRoadType: true,
+    hasBalconyProp: false,
+    hasNewBuildingProp: true,
+    hasElevatorProp: true,
+    hasNumberOfRoom: true,
+    hasNumberOfFloors: false,
+    hasFloorProp: true,
+    priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS
+  },
+  LAND: {
+    id: "LAND",
+    title: "Zemljište",
+    hasGardenSize: false,
+    hasAccesRoadType: true,
+    hasBalconyProp: false,
+    hasNewBuildingProp: false,
+    hasElevatorProp: false,
+    hasNumberOfRoom: false,
+    hasNumberOfFloors: false,
+    hasFloorProp: false,
+    priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
+  },
+  APARTMENT: {
+    id: "APARTMENT",
+    title: "Apartman",
+    hasGardenSize: false,
+    hasAccesRoadType: true,
+    hasBalconyProp: true,
+    hasNewBuildingProp: true,
+    hasElevatorProp: true,
+    hasNumberOfRoom: true,
+    hasNumberOfFloors: false,
+    hasFloorProp: true,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+  },
+  GARAGE: {
+    id: "GARAGE",
+    title: "Garaža",
+    hasGardenSize: false,
+    hasAccesRoadType: true,
+    hasBalconyProp: false,
+    hasNewBuildingProp: false,
+    hasElevatorProp: false,
+    hasNumberOfRoom: false,
+    hasNumberOfFloors: false,
+    hasFloorProp: false,
+    priceSliderOptionsSale: GARAGE_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: GARAGE_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: GARAGE_SIZE_SLIDER_OPTIONS
+  },
+  COTTAGE: {
+    id: "COTTAGE",
+    title: "Vikendica",
+    hasGardenSize: true,
+    hasAccesRoadType: true,
+    hasBalconyProp: true,
+    hasNewBuildingProp: true,
+    hasElevatorProp: false,
+    hasNumberOfRoom: true,
+    hasNumberOfFloors: true,
+    hasFloorProp: false,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
+    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
+    gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
+  }
+};
+
+const AD_STATUS = {
+  STATUS_NORMAL: 1,
+  STATUS_RESERVED: 2,
+  STATUS_SOLD: 3,
+  STATUS_DELETED: 4,
+  STATUS_URGENT: 5,
+  STATUS_DISCOUNTED: 6,
+  STATUS_RENTED: 7,
+  STATUS_VIP: 8
+};
+
+const AD_AGENCY = {
+  OLX: "OLX",
+  RENTAL: "RENTAL",
+  PROSTOR: "PROSTOR",
+  AKTIDO: "AKTIDO",
+  SALJIC: "SALJIC"
+};
+
+const CRAWLER_AD_TYPE = {
+  NONE: 0,
+  ALL: 1,
+  ONLY_SELL: 2,
+  ONLY_RENT: 3,
+  ONLY_REQUEST: 4
+};
+
+const EMAIL_FREQUENCY = {
+  ASAP: {
+    id: 1,
+    stringId: "ASAP",
+    title: "Odmah"
+  },
+  DAILY: {
+    id: 2,
+    stringId: "DAILY",
+    title: "Jednom dnevno"
+  }
+};
+
+const HEATING_TYPE = {
+  ANY: {
+    id: "ANY",
+    title: "Svi"
+  },
+  NO_HEATING: {
+    id: "NO_HEATING",
+    title: "Nije uvedeno"
+  },
+  ELECTRICITY: {
+    id: "ELECTRICITY",
+    title: "Struja"
+  },
+  GAS: {
+    id: "GAS",
+    title: "Plin"
+  },
+  WOOD: {
+    id: "WOOD",
+    title: "Drva"
+  },
+  CENTRAL_CITY: {
+    id: "CENTRAL_CITY",
+    title: "Centralno (gradsko)"
+  },
+  CENTRAL_BOILER: {
+    id: "CENTRAL_BOILER",
+    title: "Centralno (kotlovnica)"
+  },
+  CENTRAL_GAS: {
+    id: "CENTRAL_GAS",
+    title: "Centralno (plin)"
+  },
+  HEAT_PUMP: {
+    id: "HEAT_PUMP",
+    title: "Toplotna pumpa"
+  },
+  OTHER: {
+    id: "OTHER",
+    title: "Drugo"
+  }
+};
+
+const ACCESS_ROAD_TYPE = {
+  ANY: {
+    id: "ANY",
+    title: "Svi"
+  },
+  ASPHALT: {
+    id: "ASPHALT",
+    title: "Asfalt"
+  },
+  CONCRETE: {
+    id: "CONCRETE",
+    title: "Beton"
+  },
+  MACADAM: {
+    id: "MACADAM",
+    title: "Makadam"
+  },
+  OTHER: {
+    id: "OTHER",
+    title: "Drugo"
+  }
+};
+
+const FURNISHING_TYPE = {
+  NOT_FURNISHED: {
+    id: "NOT_FURNISHED",
+    title: "Nenamješten"
+  },
+  HALF_FURNISHED: {
+    id: "HALF_FURNISHED",
+    title: "Polunamješten"
+  },
+  FURNISHED: {
+    id: "FURNISHED",
+    title: "Namješten"
+  }
+};
+
+module.exports = {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_STATUS,
+  AD_AGENCY,
+  CRAWLER_AD_TYPE,
+  EMAIL_FREQUENCY,
+  HEATING_TYPE,
+  ACCESS_ROAD_TYPE,
+  FURNISHING_TYPE
+};

app/common/filterEnums.js

@@ -0,0 +1,110 @@
+const { AD_CATEGORY, ACCESS_ROAD_TYPE, HEATING_TYPE } = require("./enums");
+
+const ADVANCED_BOOLEAN_FILTERS = [
+  {
+    dbField: "balcony",
+    title: "Balkon",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+  {
+    dbField: "elevator",
+    title: "Lift",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "newBuilding",
+    title: "Novogradnja",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  }
+];
+
+const ADVANCED_SEGMENT_SELECT_FILTERS = [
+  {
+    dbField: "accessRoadType",
+    title: "Pristupni put",
+    values: Object.keys(ACCESS_ROAD_TYPE).map(key => ACCESS_ROAD_TYPE[key]),
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  }
+  // {
+  //   dbField: "heatingType",
+  //   title: "Vrsta grijanja",
+  //   values: Object.keys(HEATING_TYPE).map(key => HEATING_TYPE[key]),
+  //   categoriesToShow: [
+  //     AD_CATEGORY.FLAT,
+  //     AD_CATEGORY.HOUSE,
+  //     AD_CATEGORY.APARTMENT,
+  //     AD_CATEGORY.COTTAGE,
+  //     AD_CATEGORY.OFFICE
+  //   ]
+  // }
+];
+
+const ADVANCED_RANGE_FILTERS = [
+  {
+    id: "numberOfFloors",
+    title: "Broj spratova",
+    dbFieldMin: "numberOfFloorsMin",
+    dbFieldMax: "numberOfFloorsMax",
+    validValueMin: -1,
+    validValueMax: 50,
+    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE]
+  },
+  {
+    id: "floor",
+    title: "Sprat",
+    dbFieldMin: "floorMin",
+    dbFieldMax: "floorMax",
+    validValueMin: -10,
+    validValueMax: 50,
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    id: "numberOfRooms",
+    title: "Broj soba",
+    dbFieldMin: "numberOfRoomsMin",
+    dbFieldMax: "numberOfRoomsMax",
+    decimalPlaces: 1,
+    validValueMin: 0,
+    validValueMax: 200,
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  }
+];
+
+module.exports = {
+  ADVANCED_BOOLEAN_FILTERS,
+  ADVANCED_SEGMENT_SELECT_FILTERS,
+  ADVANCED_RANGE_FILTERS
+};

app/config/appConfig.js

@@ -0,0 +1,63 @@
+"use strict";
+require("dotenv").config({ path: __dirname + "/./../../.env" });
+
+const APP_PORT = process.env.PORT || 5000;
+const APP_BASE_URL = process.env.APP_BASE_URL || "http://localhost";
+
+const APP_URL =
+  process.env.NODE_ENV && process.env.NODE_ENV === "production"
+    ? process.env.APP_URL || "http://market-alarm"
+    : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
+
+const STAGING = process.env.ENVIRONMENT !== "production";
+
+const DEFAULT_TIMEZONE = "Europe/Sarajevo";
+
+const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
+const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
+
+const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10;
+
+const AWS_EMAIL_CONFIG = {
+  REGION: process.env.AWS_REGION || "",
+  CREDENTIALS: {
+    ACCESS_KEY_ID: process.env.AWS_KEY_ID || "",
+    SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY || ""
+  },
+  SOURCE_EMAIL: process.env.SOURCE_EMAIL || ""
+};
+
+const MAX_REAL_ESTATES_IN_EMAIL =
+  parseInt(process.env.MAX_REAL_ESTATES_IN_EMAIL) || 10;
+const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
+  parseInt(process.env.MAX_REAL_ESTATES_IN_FIRST_EMAIL) || 5;
+
+const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
+
+const API_MAP_KEY = process.env.API_MAP_KEY || "";
+
+const PROSTOR_LOGIN = {
+  EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
+  PASSWORD: process.env.PROSTOR_LOGIN_PASS
+};
+
+const USER_AGENT =
+  process.env.USER_AGENT ||
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
+
+module.exports = {
+  APP_PORT,
+  APP_URL,
+  DEFAULT_TIMEZONE,
+  CRAWLER_INTERVAL,
+  STOP_CRAWLER,
+  AWS_EMAIL_CONFIG,
+  MAX_REAL_ESTATES_IN_EMAIL,
+  MAX_REAL_ESTATES_IN_FIRST_EMAIL,
+  PRINT_CRAWLER_DEBUG,
+  API_MAP_KEY,
+  STAGING,
+  CHECK_UP_DAYS,
+  PROSTOR_LOGIN,
+  USER_AGENT
+};

app/controllers/gardenSizes.js

@@ -1,43 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-const { getRealEstateTypeEnum } = require('../helpers/enums');
-
-const getGardenSize = (req,res) => {
-
-  const unit = " m2"
-  const rangeFrom = {
-    min : 10,
-    max : 3000,
-    value : 0,
-    step : 10
-  }
-
-  const rangeTo = {
-    min : 10,
-    max : 3000,
-    value : 100,
-    step : 10
-  }
-
-  res.render('gardenSize', { rangeFrom, rangeTo, unit });
-};
-
-const postGardenSize = async (req, res) => {
-  const request = await currentRERequest(req);
-
-  const nextStepPage = req.query.nextStep || 'cijena';
-  const nextStepUrl = `/${nextStepPage}/${request.uniqueId}`;
-
-  const realEstateType = getRealEstateTypeEnum(request.realEstateType);
-  if (realEstateType && realEstateType.hasGardenSize) {
-    request.gardenSizeMin = req.body.from;
-    request.gardenSizeMax = req.body.to;
-    await request.save();
-  }
-
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getGardenSize,
-  postGardenSize
-};

app/controllers/goAgain.js

@@ -1,5 +1,6 @@
-const getGoAgain = async (req,res) => {
-  res.render('goAgain');
+const getGoAgain = async (req, res) => {
+  const title = "Uspjeh!";
+  res.render("goAgain", { title });
 };
 
 module.exports = {

app/controllers/location.js

@@ -0,0 +1,85 @@
+const { currentSearchRequest } = require("../helpers/url");
+
+const getLocation = async (req, res) => {
+  const title = "Odaberite lokaciju";
+  const nextStep = req.query.nextStep || "/";
+
+  //Check if location data already exists (active request)
+  //If it does then  get location is called through edit field query
+  //and map should show already selected location not initial map
+  let selectedLatLngBounds = {};
+  let boundsSelected = false;
+
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+  const selectedArea = searchRequest.areaToSearch;
+  const sw = selectedArea.coordinates[0][3];
+  const ne = selectedArea.coordinates[0][1];
+
+  if (sw[0] && ne[0]) {
+    selectedLatLngBounds = {
+      swLat: sw[1],
+      swLng: sw[0],
+      neLat: ne[1],
+      neLng: ne[0]
+    };
+    boundsSelected = true;
+  }
+
+  res.render("location", {
+    nextStep,
+    title,
+    boundsSelected,
+    selectedLatLngBounds
+  });
+};
+
+const postLocation = async (req, res) => {
+  let searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const northWest = [req.body.west, req.body.north];
+  const northEast = [req.body.east, req.body.north];
+  const southEast = [req.body.east, req.body.south];
+  const southWest = [req.body.west, req.body.south];
+
+  const locationInputValue =
+    req.body.locationInput && req.body.locationInput.length > 0
+      ? req.body.locationInput
+      : null;
+
+  searchRequest.areaToSearch = {
+    type: "Polygon",
+    coordinates: [[northWest, northEast, southEast, southWest, northWest]],
+    crs: { type: "name", properties: { name: "EPSG:4326" } }
+  };
+
+  let locationInputData;
+  if (req.body.locationInputData) {
+    try {
+      locationInputData = JSON.parse(req.body.locationInputData);
+    } catch (e) {
+      locationInputData = null;
+    }
+  }
+
+  await searchRequest.save();
+
+  const nextStepPage = req.query.nextStep || "filteri";
+  const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getLocation,
+  postLocation
+};

app/controllers/municipalities.js

@@ -1,26 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-const { getMunicipalitiesForRegion, getMunicipalityName } = require('../helpers/codes');
-
-const getMunicipality = async (req, res) => {
-  let request = await currentRERequest(req);
-  const municipalities = getMunicipalitiesForRegion(request.region);
-
-  res.render('municipality', { municipalities });
-};
-
-const postMunicipality = async (req, res) => {
-
-  const request = await currentRERequest(req);
-  const nextStepParam = req.query.nextStep ? "?nextStep=" + req.query.nextStep : "";
-  const nextStepUrl = `/${'naselje'}/${request.uniqueId}/${getMunicipalityName(request.region, req.body.municipality)}${nextStepParam}`;
-
-  request.municipality = req.body.municipality;
-  await request.save();
-
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getMunicipality,
-  postMunicipality
-};

app/controllers/neighborhoodMap.js

@@ -1,37 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-
-const getNeighborhood = async (req, res) => {
-    const municipality = req.params.municipality
-    const nextStep = req.query.nextStep || '/';
-    res.render('neighborhoodMap', {
-        nextStep,
-        municipality
-    });
-
-};
-
-const postNeighborhood = async (req, res) => {
-    let request = await currentRERequest(req);
-    const northWest = [req.body.west, req.body.north];
-    const northEast = [req.body.east, req.body.north];
-    const southEast = [req.body.east, req.body.south];
-    const southWest = [req.body.west, req.body.south];
-
-    request.bounding_box = {
-        type: 'Polygon', coordinates: [
-            [northWest, northEast, southEast,
-                southWest, northWest]
-        ]
-    };
-    await request.save();
-
-    const nextStepPage = req.query.nextStep || 'povrsina';
-    const nextStepUrl = `/${nextStepPage}/${request.uniqueId}`;
-
-    res.redirect(nextStepUrl);
-};
-
-module.exports = {
-    getNeighborhood,
-    postNeighborhood
-};

app/controllers/prices.js

@@ -1,40 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-
-const getPrice = (req,res) => {
-
-  const unit = " KM"
-  const rangeFrom = {
-    min : 1000,
-    max : 250000,
-    value : 0,
-    step : 1000
-  }
-
-  const rangeTo = {
-    min : 1000,
-    max : 250000,
-    value : 50000,
-    step : 1000
-  }
-
-
-  res.render('price', {rangeFrom, rangeTo, unit  });
-};
-
-const postPrice = async (req, res) => {
-  const request = await currentRERequest(req);
-
-  const nextStepPage = req.query.nextStep || 'pregled';
-  const nextStepUrl = `/${nextStepPage}/${request.uniqueId}`;
-
-  request.priceMin = req.body.from;
-  request.priceMax = req.body.to;
-  await request.save();
-
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getPrice,
-  postPrice
-};

app/controllers/queryReview.js

@@ -1,80 +1,194 @@
-const { currentRERequest } = require('../helpers/url');
-const { getRegionName, getMunicipalityName } = require('../helpers/codes');
-const { realEstateTypes, sizes, gardenSizes, prices, getEnumTypeTitle, getRealEstateTypeEnum } = require('../helpers/enums');
+const { currentSearchRequest } = require("../helpers/url");
+const { isValidEmail } = require("../helpers/email");
+const {
+  notifyForNewSearchRequest
+} = require("../services/notificationService");
+const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-const getQueryReview = async (req,res) => {
-  const request = await currentRERequest(req);
-  const nextStep = req.query.nextStep;
+const getQueryReviewTableData = searchRequest => {
+  const {
+    id,
+    adType,
+    realEstateType,
+    sizeMin,
+    sizeMax,
+    gardenSizeMin,
+    gardenSizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest.dataValues;
 
-  if (!request || !request.dataValues) {
-    return null;
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+  const enableGardenSizeEdit = realEstateTypeObject
+    ? realEstateTypeObject.hasGardenSize
+    : false;
+
+  let adTypeTitle = "";
+  switch (adType) {
+    case AD_TYPE.AD_TYPE_SALE.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
+      break;
+    case AD_TYPE.AD_TYPE_RENT.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
+      break;
+    default:
+      adTypeTitle = "-";
+      break;
   }
 
-  const { 
-    realEstateType, 
-    region, 
-    municipality, 
-    sizeMin, 
-    sizeMax, 
-    gardenSizeMin, 
-    gardenSizeMax, 
-    priceMin, 
-    priceMax } = request.dataValues;
+  const realEstateTypeTitle = realEstateTypeObject
+    ? `[${adTypeTitle}] ${realEstateTypeObject.title}`
+    : "-";
 
-  const realEstateTypeObject = getRealEstateTypeEnum(realEstateType);
-  const enableGardenSizeEdit = realEstateTypeObject ? realEstateTypeObject.hasGardenSize : false;
+  const locationTitle = "Promjenite lokaciju";
+  const sizeTitle = `${sizeMin} - ${sizeMax} m2`;
+  const gardenSizeTitle = enableGardenSizeEdit
+    ? `${gardenSizeMin} - ${gardenSizeMax} m2`
+    : "-";
+  const priceTitle = `${priceMin} - ${priceMax} KM`;
 
-  const realEstateTypeTitle = realEstateType ? getEnumTypeTitle(realEstateTypes, realEstateType) : null;
-  const regionName = region ? getRegionName(region) : null;
-  const municipalityName = (region && municipality) ? getMunicipalityName(region, municipality) : null;
-  const sizeTitle = sizeMin ? sizeMin + "-" + sizeMax + " m2" : null;
-  const gardenSizeTitle =  gardenSizeMin ? gardenSizeMin + "-" + gardenSizeMax + " m2" : null;
-  const priceTitle =  priceMin ? priceMin + "-" + priceMax + " KM" : null;
-
-  const uniqueId = request.dataValues.uniqueId ? request.dataValues.uniqueId : '';
-
-  const queryData = [
+  return [
     {
-      id: 'realEstateType',
+      id: "realEstateType",
       title: realEstateTypeTitle,
-      url: `/vrstanekretnine/${uniqueId}?nextStep=pregled`,
+      url: `/vrstanekretnine/${id}?nextStep=filteri`
     },
     {
-      id: 'region',
-      title: regionName,
-      url: `/grad/${uniqueId}?nextStep=mjesto`,
+      id: "location",
+      title: locationTitle,
+      url: `/lokacija/${id}?nextStep=pregled`
     },
     {
-      id: 'municipality',
-      title: municipalityName,
-      url: `/mjesto/${uniqueId}?nextStep=pregled`,
-    },
-    {
-      id: 'size',
+      id: "size",
       title: sizeTitle,
-      url: `/povrsina/${uniqueId}?nextStep=pregled`,
+      url: `/filteri/${id}?nextStep=pregled`
     },
     {
-      id: 'gardenSize',
+      id: "gardenSize",
       title: gardenSizeTitle,
-      url: enableGardenSizeEdit ? `/okucnica/${uniqueId}?nextStep=pregled` : '',
+      url: enableGardenSizeEdit ? `/filteri/${id}?nextStep=pregled` : ""
     },
     {
-      id: 'price',
+      id: "price",
       title: priceTitle,
-      url: `/cijena/${uniqueId}?nextStep=pregled`
+      url: `/filteri/${id}?nextStep=pregled`
     }
-  ];
+  ].filter(data => data.title != "-");
+};
 
-  res.render('queryReview', {
+const getQueryReview = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const title = "Da li je ovo to što ste tražili ?";
+  const nextStep = req.query.nextStep;
+  const error = req.query.error;
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
+  const email = searchRequest.email;
+  let selectedEmailFrequency;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.ASAP.id;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.DAILY.id;
+      break;
+  }
+
+  res.render("queryReview", {
     nextStep,
-    queryData,
+    queryReviewTableData,
+    title,
+    email,
+    selectedEmailFrequency,
+    error,
+    EMAIL_FREQUENCY
   });
 };
 
 const postQueryReview = async (req, res) => {
-  const request = await currentRERequest(req);
-  const nextStep = req.query.nextStep || `/posalji/${request.uniqueId}`;
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    return null;
+  }
+
+  const nextStep = req.query.nextStep || "/ponovo";
+  const emailFrequency =
+    parseInt(req.body.emailFrequency) || EMAIL_FREQUENCY.ASAP.id;
+  const emailInput = req.body.email;
+  const emailConfirmInput = req.body.confirmEmail;
+  const title = "Da li je ovo to što ste tražili ?";
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
+
+  let emailFrequencyStringId = EMAIL_FREQUENCY.ASAP.stringId;
+  if (emailFrequency === EMAIL_FREQUENCY.DAILY.id) {
+    emailFrequencyStringId = EMAIL_FREQUENCY.DAILY.stringId;
+  }
+
+  searchRequest.emailFrequency = emailFrequencyStringId;
+
+  if (emailInput !== emailConfirmInput) {
+    const error = "Greška ! Unešeni emailovi nisu isti";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
+
+  if (!isValidEmail(emailInput)) {
+    const error = "Greška ! Unesite validan email";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
+
+  searchRequest.email = emailInput;
+  searchRequest.subscribed = true;
+
+  try {
+    await searchRequest.save();
+  } catch (e) {
+    console.log("[ERROR] Failed to save search request !", e);
+    console.log("Search request : ", searchRequest);
+    const error =
+      "Greška ! Nismo uspjeli kreirati zahtjev za Vašu pretragu. Molimo pokuštajte ponovo";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
+
+  try {
+    await notifyForNewSearchRequest(searchRequest);
+  } catch (e) {
+    console.log("[ERROR] Failed to send initial welcome email", e);
+    console.log("Search request : ", searchRequest);
+
+    const error =
+      "Greška ! Nismo uspjeli poslati email na Vašu adresu, pokušajte sa drugom email adresom";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
 
   res.redirect(nextStep);
 };

app/controllers/querySubmit.js

@@ -1,51 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-const { isValidEmail } = require('../helpers/email');
-const { sendTemplatedEmail} = require('../helpers/awsEmail');
-
-const getQuerySubmit = async (req, res) => {
-  const nextStep = req.query.nextStep;
-  const error = req.query.error;
-
-  res.render('querySubmit', {
-    nextStep,
-    error
-  });
-};
-
-const postQuerySubmit = async (req, res) => {
-  const request = await currentRERequest(req);
-  const nextStep = req.query.nextStep || '/ponovo';
-
-  const emailInput = req.body.email;
-  const emailConfirmInput = req.body.confirm;
-  let error = "Greška ! Unesite validan email";
-
-  if (!isValidEmail(emailInput) || !isValidEmail(emailConfirmInput)) {
-
-    error = "Greška ! Unesite validan email";
-    res.render('querySubmit', {
-      error
-    });
-    return;
-  }
-
-  if (emailInput !== emailConfirmInput) {
-
-    error = "Greška ! Unešeni emailovi nisu isti";
-    res.render('querySubmit', {
-      error
-    });
-    return;
-  }
-
-  request.email = req.body.email;
-  request.subscribed = true;
-  await request.save();
-  sendTemplatedEmail(req.body.email, request);
-  res.redirect(nextStep); 
-};
-
-module.exports = {
-  getQuerySubmit,
-  postQuerySubmit
-};

app/controllers/realEstateFilters.js

@@ -0,0 +1,247 @@
+const { currentSearchRequest } = require("../helpers/url");
+const { AD_CATEGORY, AD_TYPE, ACCESS_ROAD_TYPE } = require("../common/enums");
+const {
+  ADVANCED_BOOLEAN_FILTERS,
+  ADVANCED_SEGMENT_SELECT_FILTERS,
+  ADVANCED_RANGE_FILTERS
+} = require("../common/filterEnums");
+
+const getFilters = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const title = "Filteri za pretraživanje";
+
+  const {
+    adType,
+    realEstateType,
+    priceMin,
+    priceMax,
+    sizeMin,
+    sizeMax,
+    gardenSizeMin,
+    gardenSizeMax,
+    numberOfRoomsMin,
+    numberOfRoomsMax,
+    numberOfFloorsMin,
+    numberOfFloorsMax,
+    floorMin,
+    floorMax,
+    includeIncompleteAds,
+    balcony,
+    elevator,
+    newBuilding,
+    accessRoadType,
+    includeWithoutPrice
+  } = searchRequest;
+  const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
+
+  const {
+    hasGardenSize,
+    priceSliderOptionsSale,
+    priceSliderOptionsRent,
+    sizeSliderOptions,
+    gardenSizeSliderOptions
+  } = category;
+
+  let priceSliderOptions;
+  if (adType === AD_TYPE.AD_TYPE_SALE.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsSale);
+  } else if (adType === AD_TYPE.AD_TYPE_RENT.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsRent);
+  } else {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  // TODO: Maybe this is slow, pay attention to this
+  const filterFilters = filterObject => {
+    const filterCategories = filterObject.categoriesToShow;
+    return filterCategories.indexOf(category) !== -1;
+  };
+
+  const advancedBooleanFilterObjects = ADVANCED_BOOLEAN_FILTERS.filter(
+    filterFilters
+  );
+  const advancedSegmentSelectFilterObjects = ADVANCED_SEGMENT_SELECT_FILTERS.filter(
+    filterFilters
+  );
+  const advancedRangeFilterObjects = ADVANCED_RANGE_FILTERS.filter(
+    filterFilters
+  );
+
+  const advancedBooleanFilterValues = {
+    includeIncompleteAds,
+    balcony,
+    elevator,
+    newBuilding
+  };
+  const advancedSegmentSelectFilterValues = {
+    accessRoadType
+  };
+  const advancedRangeFilterValues = {
+    numberOfFloorsMin,
+    numberOfFloorsMax,
+    numberOfRoomsMin,
+    numberOfRoomsMax,
+    floorMin,
+    floorMax
+  };
+
+  if (priceMin || priceMax) {
+    priceSliderOptions.start = [priceMin, priceMax];
+  }
+
+  if (sizeMin || sizeMax) {
+    sizeSliderOptions.start = [sizeMin, sizeMax];
+  }
+
+  if (gardenSizeSliderOptions && (gardenSizeMin || gardenSizeMax)) {
+    gardenSizeSliderOptions.start = [gardenSizeMin, gardenSizeMax];
+  }
+
+  res.render("realEstateFilters", {
+    title,
+    hasGardenSize,
+    priceSliderOptions: JSON.stringify(priceSliderOptions),
+    sizeSliderOptions: JSON.stringify(sizeSliderOptions),
+    gardenSizeSliderOptions: JSON.stringify(gardenSizeSliderOptions),
+    advancedBooleanFilterObjects,
+    advancedBooleanFilterValues,
+    advancedSegmentSelectFilterObjects,
+    advancedSegmentSelectFilterValues,
+    advancedRangeFilterObjects,
+    advancedRangeFilterValues,
+    includeIncompleteAds,
+    includeWithoutPrice
+  });
+};
+
+const postFilters = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
+
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const nextStepPage = req.query.nextStep || "pregled";
+  const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+
+  const priceMin = parseInt(req.body.priceMin) || 0;
+  const priceMax = parseInt(req.body.priceMax) || 0;
+  const sizeMin = parseInt(req.body.sizeMin) || 0;
+  const sizeMax = parseInt(req.body.sizeMax) || 0;
+
+  const advancedRangeFilters = {};
+
+  ADVANCED_RANGE_FILTERS.forEach(filter => {
+    let parsingFunction = parseInt;
+    if (filter.decimalPlaces) {
+      parsingFunction = parseFloat;
+    }
+
+    advancedRangeFilters[filter.dbFieldMin] = parsingFunction(
+      req.body[filter.dbFieldMin]
+    );
+    advancedRangeFilters[filter.dbFieldMax] = parsingFunction(
+      req.body[filter.dbFieldMax]
+    );
+
+    advancedRangeFilters[filter.dbFieldMin] = isNaN(
+      advancedRangeFilters[filter.dbFieldMin]
+    )
+      ? null
+      : advancedRangeFilters[filter.dbFieldMin];
+    advancedRangeFilters[filter.dbFieldMax] = isNaN(
+      advancedRangeFilters[filter.dbFieldMax]
+    )
+      ? null
+      : advancedRangeFilters[filter.dbFieldMax];
+
+    try {
+      if (filter.decimalPlaces) {
+        advancedRangeFilters[filter.dbFieldMin] = advancedRangeFilters[
+          filter.dbFieldMin
+        ].toFixed(filter.decimalPlaces);
+        advancedRangeFilters[filter.dbFieldMax] = advancedRangeFilters[
+          filter.dbFieldMax
+        ].toFixed(filter.decimalPlaces);
+      }
+    } catch (e) {
+      advancedRangeFilters[filter.dbFieldMin] = null;
+      advancedRangeFilters[filter.dbFieldMax] = null;
+    }
+
+    if (
+      advancedRangeFilters[filter.dbFieldMin] < filter.validValueMin ||
+      advancedRangeFilters[filter.dbFieldMin] > filter.validValueMax
+    ) {
+      advancedRangeFilters[filter.dbFieldMin] = filter.validValueMin;
+    }
+
+    if (
+      advancedRangeFilters[filter.dbFieldMax] < filter.validValueMin ||
+      advancedRangeFilters[filter.dbFieldMax] > filter.validValueMax
+    ) {
+      advancedRangeFilters[filter.dbFieldMax] = filter.validValueMax;
+    }
+  });
+
+  const includeIncompleteAds = req.body.includeIncompleteAds === "on";
+  const includeWithoutPrice = req.body.includeWithoutPrice === "on";
+
+  const balcony = req.body.balcony === "on";
+  const elevator = req.body.elevator === "on";
+  const newBuilding = req.body.newBuilding === "on";
+
+  const accessRoadType = req.body.accessRoadType;
+  if (!ACCESS_ROAD_TYPE[accessRoadType]) {
+    res.render("notFound", { title: " Greška !" });
+    return;
+  }
+
+  //TODO: Filter validation
+  searchRequest.priceMin = priceMin;
+  searchRequest.priceMax = priceMax;
+  searchRequest.sizeMin = sizeMin;
+  searchRequest.sizeMax = sizeMax;
+
+  for (const filter of Object.keys(advancedRangeFilters)) {
+    searchRequest[filter] = advancedRangeFilters[filter];
+  }
+
+  searchRequest.balcony = balcony;
+  searchRequest.elevator = elevator;
+  searchRequest.newBuilding = newBuilding;
+
+  searchRequest.includeIncompleteAds = includeIncompleteAds;
+  searchRequest.includeWithoutPrice = includeWithoutPrice;
+
+  searchRequest.accessRoadType = accessRoadType;
+
+  if (
+    req.body.gardenSizeMin !== undefined &&
+    req.body.gardenSizeMax !== undefined
+  ) {
+    const gardenSizeMin = parseInt(req.body.gardenSizeMin);
+    const gardenSizeMax = parseInt(req.body.gardenSizeMax);
+
+    //TODO: Filter validation
+
+    searchRequest.gardenSizeMin = gardenSizeMin;
+    searchRequest.gardenSizeMax = gardenSizeMax;
+  }
+  await searchRequest.save();
+
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getFilters,
+  postFilters
+};

app/controllers/realEstateTypes.js

@@ -1,41 +1,78 @@
-const db = require('../models/index');
+const { currentSearchRequest } = require("../helpers/url");
+const { createSearchRequest } = require("../helpers/db/searchRequest");
 
-const { currentRERequest } = require('../helpers/url');
-const { realEstateTypes, getRealEstateTypeEnum } = require('../helpers/enums');
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
 
+const getRealEstateTypes = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
 
-const getRealEstateTypes = (req,res) => {
-  res.render('realEstateType', { realEstateTypes });
+  const title = "Koju nekretninu tražite?";
+  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
+  if (
+    searchRequest &&
+    searchRequest.adType &&
+    searchRequest.adType === AD_TYPE.AD_TYPE_RENT.stringId
+  ) {
+    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
+  }
+  const realEstateTypes = Object.keys(AD_CATEGORY)
+    .map(category => AD_CATEGORY[category])
+    .filter(category => category.title);
+
+  res.render("realEstateType", {
+    selectedAdType,
+    realEstateTypes,
+    title,
+    AD_TYPE
+  });
 };
 
 const postRealEstateTypes = async (req, res) => {
-  const request = await currentRERequest(req);
+  const searchRequest = await currentSearchRequest(req);
 
-  const nextStepPage = req.query.nextStep || 'grad';
+  const adType = parseInt(req.body.adType);
 
-  if (request && request.uniqueId) {
-    const nextStepUrl = `/${nextStepPage}/${request.uniqueId}`;
-    request.realEstateType = req.body.realestatetype;
-    if (!getRealEstateTypeEnum(request.realEstateType).hasGardenSize){
-      request.gardenSize = null;
-    }
-    await request.save();
+  const adTypeStringIds = {
+    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
+    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
+  };
 
-    res.redirect(nextStepUrl)
-  } else {
-    db.RealEstateRequest.create({
-      realEstateType: req.body.realestatetype
-    }).then( (result) => {
-      const nextStepUrl = `/${nextStepPage}/${result.uniqueId}`;
-      res.redirect(nextStepUrl);
-    }).catch( (e) => {
-      res.send(e);
-    });
+  const adTypeStringId =
+    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
+
+  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
+    category => !!AD_CATEGORY[category].title
+  );
+
+  const selectedRealEstateType = req.body.realEstateType || null;
+  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
+    res.render("notFound", { title: " " });
+    return;
   }
 
+  const nextStepPage = req.query.nextStep || "lokacija";
 
+  let nextStepUrl = "";
+  if (searchRequest && searchRequest.id) {
+    nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+    searchRequest.adType = adTypeStringId;
+    searchRequest.realEstateType = selectedRealEstateType;
 
+    await searchRequest.save();
+  } else {
+    try {
+      const newSearchRequest = await createSearchRequest({
+        adType: adTypeStringId,
+        realEstateType: selectedRealEstateType
+      });
 
+      nextStepUrl = `/${nextStepPage}/${newSearchRequest.id}`;
+    } catch (error) {
+      console.log(error);
+      nextStepUrl = `/`;
+    }
+  }
+  res.redirect(nextStepUrl);
 };
 
 module.exports = {

app/controllers/realEstates.js

@@ -0,0 +1,17 @@
+"use strict";
+const {
+  findRealEstatesForSearchRequest
+} = require("../helpers/db/searchRequestMatch");
+const { AD_STATUS } = require("../common/enums");
+
+const getRealEstates = async (req, res) => {
+  const searchRequestId = req.params["searchRequestId"] || "";
+  const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
+
+  const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
+  res.render("realEstates", { realEstates, title, AD_STATUS });
+};
+
+module.exports = {
+  getRealEstates
+};

app/controllers/redirect.js

@@ -0,0 +1,36 @@
+const { getRealEstateById } = require("../helpers/db/realEstate");
+const { AD_STATUS } = require("../common/enums");
+
+const getRedirect = async (req, res) => {
+  const id = req.params.id || null;
+  let error = false;
+  let redirectUrl = undefined;
+  let vipAd = undefined;
+  if (!id) {
+    error = true;
+  } else {
+    try {
+      const realEstate = await getRealEstateById(id);
+      if (!realEstate) {
+        error = true;
+      } else {
+        redirectUrl = realEstate.url;
+        vipAd = realEstate.adStatus === AD_STATUS.STATUS_VIP;
+      }
+    } catch (e) {
+      error = true;
+    }
+  }
+
+  if (error) {
+    const title = "";
+    res.render("notFound", { title });
+  } else {
+    const title = "Preusmjeravanje";
+    res.render("redirect", { title, redirectUrl, vipAd });
+  }
+};
+
+module.exports = {
+  getRedirect
+};

app/controllers/regions.js

@@ -1,27 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-const { getRegions } = require('../helpers/codes');
-
-const regions = getRegions();
-
-const getRegion = (req,res) => {
-  res.render('region', { regions });
-};
-
-const postRegion = async (req, res) => {
-  const request = await currentRERequest(req);
-
-  const nextStepQueryParam = req.query.nextStep ? '?nextStep=pregled' : '';
-  const nextStepPage = req.query.nextStep || 'mjesto';
-  const nextStepUrl = `/${nextStepPage}/${request.uniqueId}${nextStepQueryParam}`;
-
-  request.region = req.body.region;
-  request.municipality = null;
-  await request.save();
-
-  res.redirect(nextStepUrl)
-};
-
-module.exports = {
-  getRegion,
-  postRegion
-};

app/controllers/sizes.js

@@ -1,42 +0,0 @@
-const { currentRERequest } = require('../helpers/url');
-const { sizes, getRealEstateTypeEnum } = require('../helpers/enums');
-
-const getSize = (req,res) => {
-
-  const unit = " m2"
-  const rangeFrom = {
-    min : 10,
-    max : 250,
-    value : 0,
-    step : 10
-  }
-
-  const rangeTo = {
-    min : 10,
-    max : 250,
-    value : 50,
-    step : 10
-  }
-
-  res.render('size', { rangeFrom, rangeTo, unit });
-};
-
-const postSize = async (req, res) => {
-  const request = await currentRERequest(req);
-
-  const realEstateType = getRealEstateTypeEnum(request.realEstateType);
-
-  const nextStep = realEstateType && realEstateType.hasGardenSize ? 'okucnica' : 'cijena';
-  const nextStepPage = req.query.nextStep || nextStep;
-  const nextStepUrl = `/${nextStepPage}/${request.uniqueId}`;
-  request.sizeMin = req.body.from;
-  request.sizeMax = req.body.to;
-  await request.save();
-
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getSize,
-  postSize
-};

app/controllers/unsubscribe.js

@@ -1,15 +1,22 @@
-
-const { currentRERequest } = require('../helpers/url');
+const { currentSearchRequest } = require("../helpers/url");
 
 const getUnsubscribe = async (req, res) => {
-    const request = await currentRERequest(req);
-    request.subscribed = false;
-    await request.save();
+  const title = "Uspješno ste se odjavili";
+  const searchRequest = await currentSearchRequest(req);
 
-    res.render('unsubscribe', { nextStep: '/vrstanekretnine' });
+  if (!searchRequest || !searchRequest.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  searchRequest.subscribed = false;
+  searchRequest.deletedEmail = searchRequest.email;
+  searchRequest.email = "";
+  await searchRequest.save();
+
+  res.render("unsubscribe", { nextStep: "/vrstanekretnine", title });
 };
 
-
 module.exports = {
-    getUnsubscribe
-};
+  getUnsubscribe
+};

app/controllers/welcome.js

@@ -1,7 +1,42 @@
-const getWelcome = (req,res) => {
-  res.render('welcome', { nextStep: '/vrstanekretnine' } );
+const { createSearchRequest } = require("../helpers/db/searchRequest");
+
+const { AD_TYPE, AD_CATEGORY } = require("../common/enums");
+
+const getWelcome = (req, res) => {
+  res.render("welcome", {
+    title: false,
+    AD_TYPE
+  });
+};
+
+const postWelcome = async (req, res) => {
+  const adType = parseInt(req.body.adType);
+
+  const adTypeStringIds = {
+    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
+    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
+  };
+
+  const adTypeStringId =
+    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
+
+  let nextStepUrl = "";
+  try {
+    const newSearchRequest = await createSearchRequest({
+      adType: adTypeStringId,
+      realEstateType: AD_CATEGORY.FLAT.id
+    });
+
+    nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
+  } catch (error) {
+    console.log(error);
+    nextStepUrl = `/`;
+  }
+
+  res.redirect(nextStepUrl);
 };
 
 module.exports = {
-  getWelcome
+  getWelcome,
+  postWelcome
 };

app/crawler/crawl.js

@@ -0,0 +1,92 @@
+"use strict";
+/*
+   Entry point for crawling functionality
+   All communication between crawlers and savers is here
+   All environment specific configuration is read here and
+   passed to the crawlers and savers.
+*/
+const OlxCrawler = require("./specificCrawlers/olx");
+const RentalCrawler = require("./specificCrawlers/rental");
+const ProstorCrawler = require("./specificCrawlers/prostor");
+const AktidoCrawler = require("./specificCrawlers/aktido");
+const SaljicCrawler = require("./specificCrawlers/saljic");
+
+const {
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
+} = require("./crawlerConfig");
+const PostgresSaver = require("./savers/postgres");
+
+async function crawlAll() {
+  const postgresSaver = new PostgresSaver();
+
+  const crawlers = [
+    new OlxCrawler(
+      [postgresSaver],
+      OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
+      OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
+      OLX_CONFIG.OLX_MAX_PAGES,
+      OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+      OLX_CONFIG.OLX_IGNORED_USERNAMES,
+      OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
+    ),
+    new RentalCrawler(
+      [postgresSaver],
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
+      RENTAL_CONFIG.RENTAL_MAX_PAGES,
+      RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
+      RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
+      RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
+    ),
+    new ProstorCrawler(
+      [postgresSaver],
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
+      PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
+      PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
+      PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
+      PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
+    ),
+    new AktidoCrawler(
+      [postgresSaver],
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
+      AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
+      AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
+      AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
+      AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
+    ),
+    new SaljicCrawler(
+      [postgresSaver],
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
+      SALJIC_CONFIG.SALJIC_MAX_PAGES,
+      SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
+      SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
+      SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
+    )
+  ];
+
+  const newRealEstates = [];
+
+  for (const crawler of crawlers) {
+    try {
+      const newRealEstatesFromSingleCrawler = await crawler.crawl();
+      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
+        newRealEstates.push(...newRealEstatesFromSingleCrawler);
+      }
+    } catch (e) {
+      console.log("Error crawling. Trying next crawler! ", e);
+    }
+  }
+
+  return newRealEstates;
+}
+
+module.exports = {
+  crawlAll
+};

app/crawler/crawlerConfig.js

@@ -0,0 +1,16 @@
+"use strict";
+require("dotenv").config({ path: __dirname + "/./../../.env" });
+
+const OLX_CONFIG = require("./specificConfigs/olx");
+const RENTAL_CONFIG = require("./specificConfigs/rental");
+const PROSTOR_CONFIG = require("./specificConfigs/prostor");
+const AKTIDO_CONFIG = require("./specificConfigs/aktido");
+const SALJIC_CONFIG = require("./specificConfigs/saljic");
+
+module.exports = {
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
+};

app/crawler/npmCrawl.js

@@ -0,0 +1,5 @@
+const { crawlAll } = require("./crawl");
+
+(async () => {
+  await crawlAll();
+})();

app/crawler/savers/postgres.js

@@ -0,0 +1,63 @@
+const moment = require("moment");
+
+const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
+const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory");
+
+class PostgresSaver {
+  connect() {
+    //TODO: It seems we never worry about open/close connection with Sequelize ?
+    //TODO: Check if postgres is ready
+    return true;
+  }
+
+  async save(results) {
+    const savedRecords = await bulkUpsertRealEstates(results);
+    //Extruding data for price history table
+    const resultPrices = savedRecords.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+    const savedPrices = await bulkUpsertPriceHistory(resultPrices);
+
+    if (Array.isArray(savedRecords)) {
+      const newRealEstates = [];
+      const existingRealEstates = [];
+
+      for (const savedRecord of savedRecords) {
+        const { createdAt, updatedAt } = savedRecord;
+
+        const createdAtMoment = moment.utc(createdAt);
+        const updatedAtMoment = moment.utc(updatedAt);
+
+        if (createdAtMoment.isSame(updatedAtMoment, "second")) {
+          newRealEstates.push(savedRecord);
+        } else {
+          existingRealEstates.push(savedRecord);
+        }
+      }
+
+      return {
+        newRecords: newRealEstates,
+        existingRecords: existingRealEstates
+      };
+    } else {
+      throw { message: "[POSTGRES] Failed to save records" };
+    }
+  }
+
+  close() {
+    //TODO: It seems we never worry about open/close connection with Sequelize ?
+    return true;
+  }
+}
+
+module.exports = PostgresSaver;

app/crawler/specificConfigs/aktido.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const aktidoCrawlerAdType =
+  process.env.AKTIDO_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.AKTIDO_CRAWLER_AD_TYPE]
+    : null;
+
+const aktidoParsedCrawlerAdCategories =
+  process.env.AKTIDO_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.AKTIDO_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const aktidoIgnoredUsernames = [];
+
+const transformedAktidoCrawlerAdCategories = aktidoParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  AKTIDO_MAX_PAGES: parseInt(process.env.AKTIDO_MAX_PAGES) || 500,
+  AKTIDO_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.AKTIDO_MAX_RESULTS_PER_PAGE) || 50,
+  AKTIDO_CRAWLER_AD_TYPE: aktidoCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
+  AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
+  AKTIDO_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
+  AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
+};

app/crawler/specificConfigs/olx.js

@@ -0,0 +1,39 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const olxCrawlerAdType =
+  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
+    : null;
+
+const olxParsedCrawlerAdCategories =
+  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const olxIgnoredUsernames =
+  process.env.OLX_IGNORED_USERNAMES !== undefined
+    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
+        username.trim()
+      )
+    : [];
+
+const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
+  OLX_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
+  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
+  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
+  OLX_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
+  OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
+};

app/crawler/specificConfigs/prostor.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const prostorCrawlerAdType =
+  process.env.PROSTOR_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.PROSTOR_CRAWLER_AD_TYPE]
+    : null;
+
+const prostorParsedCrawlerAdCategories =
+  process.env.PROSTOR_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.PROSTOR_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const prostorIgnoredUsernames = [];
+
+const transformedProstorCrawlerAdCategories = prostorParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  PROSTOR_MAX_PAGES: parseInt(process.env.PROSTOR_MAX_PAGES) || 100,
+  PROSTOR_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 5000,
+  PROSTOR_CRAWLER_AD_TYPE: prostorCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  PROSTOR_CRAWLER_AD_CATEGORIES: transformedProstorCrawlerAdCategories,
+  PROSTOR_IGNORED_USERNAMES: prostorIgnoredUsernames || [],
+  PROSTOR_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.PROSTOR_DELAY_BETWEEN_PAGES) || 1000,
+  PROSTOR_FORCE_CRAWL: !!parseInt(process.env.PROSTOR_FORCE_CRAWL)
+};

app/crawler/specificConfigs/rental.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const rentalCrawlerAdType =
+  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
+    : null;
+
+const rentalParsedCrawlerAdCategories =
+  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const rentalIgnoredUsernames = [];
+
+const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
+  RENTAL_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
+  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
+  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
+  RENTAL_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000,
+  RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL)
+};

app/crawler/specificConfigs/saljic.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const saljicCrawlerAdType =
+  process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
+    : null;
+
+const saljicParsedCrawlerAdCategories =
+  process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const saljicIgnoredUsernames = [];
+
+const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
+  SALJIC_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
+  SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
+  SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
+  SALJIC_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
+  SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
+};

app/crawler/specificCrawlers/aktido.js

@@ -0,0 +1,846 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  HEATING_TYPE,
+  ACCESS_ROAD_TYPE,
+  FURNISHING_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const AKTIDO_ENUMS = {
+  AKTIDO_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  AKTIDO_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  AKTIDO_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
+
+class AktidoCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.aktido.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !AKTIDO_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = AKTIDO_ENUMS.AKTIDO_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = AKTIDO_ENUMS.AKTIDO_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[AKTIDO] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".moreInfo")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[AKTIDO] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        let adStatus = AD_STATUS.STATUS_NORMAL;
+        const aktidoId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromAktidoId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromAktidoActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const descriptionIds = extractedData["re_descriptions_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(descriptionIds)) {
+          throw {
+            message:
+              'Expected array od descriptions but "re_descriptions_id" not found !'
+          };
+        }
+
+        const spaceIds = extractedData["re_spaces_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(spaceIds)) {
+          throw {
+            message: 'Expected array od spaces but "re_spaces_id" not found !'
+          };
+        }
+
+        const infrastructureIds = extractedData["re_infrastructure_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(infrastructureIds)) {
+          throw {
+            message:
+              'Expected array od infrastructures but "re_infrastructure_id" not found !'
+          };
+        }
+
+        const floorNoIds = extractedData["re_floorNO_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(floorNoIds)) {
+          throw {
+            message:
+              'Expected array od infrastructures but "re_floorNO_id" not found !'
+          };
+        }
+
+        // counting floor enums
+        // for (let i = 1; i < 10; i++) {
+        //   const floorEnumsTitle = $(
+        //     `body > div.container-fluid > div.container > div:nth-child(2) > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div:nth-child(1) > div > div > div.col-xs-12.col-sm-4.box-details > div.body > p:nth-child(${i}) > span:nth-child(1)`
+        //   )
+        //     .text()
+        //     .trim();
+        //   if (floorEnumsTitle === "Spratnost:") {
+        //     const floorEnumsValue = $(
+        //       `body > div.container-fluid > div.container > div:nth-child(2) > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div:nth-child(1) > div > div > div.col-xs-12.col-sm-4.box-details > div.body > p:nth-child(${i}) > span:nth-child(2)`
+        //     )
+        //       .text()
+        //       .trim()
+        //       .split(",");
+        //
+        //     console.log("==========");
+        //     floorNoIds.forEach((id, index) => {
+        //       console.log("\t", id, " = ", floorEnumsValue[index]);
+        //     });
+        //     break;
+        //   }
+        // }
+
+        // enumerating infrastructure - relation between id and infrastructure title
+        // let found = false;
+        // let infrastructureDescriptions = {};
+        // for (let i = 1; i < 5; i++) {
+        //   found = false;
+        //   for (let j = 1; j < 10; j++) {
+        //     const infrastructureTitle = $(
+        //       `#b2 > div > div:nth-child(${i}) > div > ul > li:nth-child(${j}) > strong`
+        //     )
+        //       .text()
+        //       .trim();
+        //     if (infrastructureTitle === "Osnovna infrastruktura:") {
+        //       found = true;
+        //
+        //       const infrastructureValues = $(
+        //         `#b2 > div > div:nth-child(${i}) > div > ul > li:nth-child(${j}) > div`
+        //       )
+        //         .text()
+        //         .trim()
+        //         .split(",");
+        //
+        //       infrastructureIds.forEach((id, index) => {
+        //         infrastructureDescriptions[id] = infrastructureValues[index];
+        //       });
+        //     }
+        //   }
+        //   if (found) {
+        //     break;
+        //   }
+        // }
+
+        const realEstatePropertiesFromDescriptions = this.getPropertiesFromDescriptions(
+          descriptionIds
+        );
+        const realEstatePropertiesFromSpaces = this.getPropertiesFromSpaces(
+          spaceIds
+        );
+
+        const realEstatePropertiesFromInfrastructure = this.getPropertiesFromInfrastructure(
+          infrastructureIds
+        );
+
+        if (extractedData["adm_realEstates_discount"] === "1") {
+          adStatus = AD_STATUS.STATUS_DISCOUNTED;
+        }
+
+        let numberOfRooms =
+            parseInt(extractedData["re_realEstates_roomsNO"]) +
+              parseInt(extractedData["re_realEstates_bedroomNO"]) || null,
+          numberOfFloors =
+            parseInt(extractedData["re_realEstates_floorsNO"]) ||
+            this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]),
+          floor =
+            parseInt(extractedData["re_realEstates_floorNO"]) ||
+            this.getFloorNumberFromFloorId(extractedData["re_floorNO_id"]),
+          accessRoadType = realEstatePropertiesFromDescriptions.accessRoadType,
+          heatingType =
+            this.getHeatingTypeId(extractedData["re_heating_id"]) || null,
+          furnishingType = realEstatePropertiesFromDescriptions.furnishingType,
+          balcony =
+            realEstatePropertiesFromDescriptions.balcony ||
+            realEstatePropertiesFromSpaces.balcony,
+          newBuilding = extractedData["op_realEstates_newBuilding"]
+            ? extractedData["op_realEstates_newBuilding"] === "1"
+            : null,
+          elevator = realEstatePropertiesFromDescriptions.elevator,
+          water =
+            realEstatePropertiesFromDescriptions.water ||
+            realEstatePropertiesFromInfrastructure.water,
+          electricity =
+            realEstatePropertiesFromDescriptions.electricity ||
+            realEstatePropertiesFromInfrastructure.electricity,
+          drainageSystem =
+            realEstatePropertiesFromInfrastructure.drainageSystem,
+          registeredInZkBooks =
+            extractedData["op_realEstates_ownerPermit"] === 1 || null,
+          recentlyAdapted = null,
+          parking =
+            realEstatePropertiesFromDescriptions.parking ||
+            realEstatePropertiesFromSpaces.parking,
+          garage = realEstatePropertiesFromSpaces.garage,
+          gas = realEstatePropertiesFromInfrastructure.gas,
+          antiTheftDoor = realEstatePropertiesFromDescriptions.antiTheftDoor,
+          airCondition = realEstatePropertiesFromDescriptions.airCondition,
+          phoneConnection =
+            realEstatePropertiesFromInfrastructure.phoneConnection,
+          cableTV = realEstatePropertiesFromInfrastructure.cableTV,
+          internet = realEstatePropertiesFromInfrastructure.internet,
+          basementAttic = realEstatePropertiesFromSpaces.basementAttic,
+          storeRoom = realEstatePropertiesFromSpaces.storeRoom,
+          videoSurveillance =
+            realEstatePropertiesFromDescriptions.videoSurveillance ||
+            realEstatePropertiesFromInfrastructure.videoSurveillance,
+          alarm = realEstatePropertiesFromDescriptions.alarm,
+          suitableForStudents = null,
+          includingBills =
+            extractedData["op_realEstates_utilitiesIncluded"] === "1" || null,
+          animalsAllowed = null,
+          pool = realEstatePropertiesFromDescriptions.pool,
+          urbanPlanPermit =
+            extractedData["op_realEstates_locationPermit"] === "1" ||
+            realEstatePropertiesFromDescriptions.urbanPlanPermit,
+          buildingPermit =
+            extractedData["op_realEstates_buildingPermit"] === "1" || null,
+          utilityConnection =
+            realEstatePropertiesFromDescriptions.utilityConnection,
+          distanceToRiver = null,
+          numberOfViewsAgency = null;
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          AKTIDO_ENUMS.AKTIDO_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          AKTIDO_ENUMS.AKTIDO_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const data = {
+          url,
+          agencyObjectId: aktidoId,
+          originAgencyName: AD_AGENCY.AKTIDO,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString(),
+          numberOfRooms,
+          numberOfFloors,
+          floor,
+          accessRoadType,
+          heatingType,
+          furnishingType,
+          balcony,
+          newBuilding,
+          elevator,
+          water,
+          electricity,
+          drainageSystem,
+          registeredInZkBooks,
+          recentlyAdapted,
+          parking,
+          garage,
+          gas,
+          antiTheftDoor,
+          airCondition,
+          phoneConnection,
+          cableTV,
+          internet,
+          basementAttic,
+          storeRoom,
+          videoSurveillance,
+          alarm,
+          suitableForStudents,
+          includingBills,
+          animalsAllowed,
+          pool,
+          urbanPlanPermit,
+          buildingPermit,
+          utilityConnection,
+          distanceToRiver,
+          numberOfViewsAgency
+        };
+
+        return data;
+      } else {
+        console.log("[AKTIDO] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromAktidoId(aktidoCategoryId) {
+    switch (aktidoCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromAktidoActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  getPropertiesFromDescriptions(descriptionIds) {
+    const result = {
+      accessRoadType: null,
+      furnishingType: null,
+      balcony: null,
+      elevator: null,
+      parking: null,
+      antiTheftDoor: null,
+      airCondition: null,
+      videoSurveillance: null,
+      alarm: null,
+      pool: null,
+      urbanPlanPermit: null,
+      utilityConnection: null,
+      water: null,
+      electricity: null
+    };
+
+    for (const descriptionId of descriptionIds) {
+      switch (descriptionId) {
+        case 16:
+          result.furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
+          break;
+        case 17:
+          result.furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
+          break;
+        case 1:
+        case 28:
+          result.furnishingType = FURNISHING_TYPE.FURNISHED.id;
+          break;
+        case 14:
+          result.elevator = true;
+          break;
+        case 39:
+          result.electricity = true;
+          break;
+        case 40:
+          result.water = true;
+          break;
+        case 41:
+        case 58:
+          result.accessRoadType = ACCESS_ROAD_TYPE.ASPHALT.id;
+          break;
+        case 26:
+          result.balcony = true;
+          break;
+        case 62:
+          result.parking = true;
+          break;
+        case 3:
+          result.antiTheftDoor = true;
+          break;
+        case 2:
+        case 21:
+          result.airCondition = true;
+          break;
+        case 4:
+          result.alarm = true;
+          break;
+        case 55:
+          result.videoSurveillance = true;
+          break;
+        case 9:
+          result.pool = true;
+          break;
+        case 60:
+          result.urbanPlanPermit = true;
+          break;
+        case 38:
+          result.utilityConnection = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getPropertiesFromSpaces(spaceIds) {
+    const result = {
+      balcony: null,
+      parking: null,
+      garage: null,
+      basementAttic: null,
+      storeRoom: null
+    };
+
+    for (const spaceId of spaceIds) {
+      switch (spaceId) {
+        case 36:
+        case 12:
+          result.parking = true;
+          break;
+        case 1:
+        case 2:
+        case 3:
+          result.balcony = true;
+          break;
+        case 4:
+        case 30:
+          result.garage = true;
+          break;
+        case 9:
+        case 10:
+          result.storeRoom = true;
+          break;
+        case 18:
+        case 34:
+        case 37:
+        case 27:
+          result.basementAttic = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getHeatingTypeId(heatingRentalId) {
+    // heatingRentalId can have multiple values, like: "1, 2, 3", parseInt will take first integer value
+    const heatingId = parseInt(heatingRentalId);
+    switch (heatingId) {
+      case 27:
+      case 16:
+        return HEATING_TYPE.GAS.id;
+      case 4:
+        return HEATING_TYPE.CENTRAL_GAS.id;
+      case 3:
+      case 23:
+      case 6:
+      case 7:
+      case 8:
+      case 9:
+      case 10:
+        return HEATING_TYPE.CENTRAL_BOILER.id;
+      case 2:
+      case 13:
+      case 30:
+      case 17:
+      case 29:
+      case 31:
+        return HEATING_TYPE.ELECTRICITY.id;
+      case 24:
+      case 25:
+      case 12:
+        return HEATING_TYPE.CENTRAL_CITY.id;
+      case 26:
+      case 21:
+      case 20:
+        return HEATING_TYPE.WOOD.id;
+      case 28:
+      case 19:
+        return HEATING_TYPE.HEAT_PUMP.id;
+      case 14:
+      case 32:
+        return HEATING_TYPE.OTHER.id;
+      default:
+        return null;
+    }
+  }
+
+  getPropertiesFromInfrastructure(infrastructureIds) {
+    const result = {
+      electricity: null,
+      water: null,
+      gas: null,
+      drainageSystem: null,
+      phoneConnection: null,
+      internet: null,
+      videoSurveillance: null,
+      cableTV: null
+    };
+
+    for (const infrastructureId of infrastructureIds) {
+      switch (infrastructureId) {
+        case 1:
+          result.electricity = true;
+          break;
+        case 2:
+          result.water = true;
+          break;
+        case 4:
+          result.gas = true;
+          break;
+        case 5:
+          result.drainageSystem = true;
+          break;
+        case 7:
+        case 8:
+          result.phoneConnection = true;
+          break;
+        case 10:
+          result.internet = true;
+          break;
+        case 11:
+          result.cableTV = true;
+          break;
+        case 16:
+        case 17:
+          result.videoSurveillance = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getFloorNumberFromFloorId(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+    // just extracting floor number from first element
+
+    const floorsId = floorsIdText.split(",");
+    if (floorsId.length === 0) {
+      return null;
+    }
+
+    const firstFloorId = parseInt(floorsId[0]);
+
+    // 1 pod
+    // 2 sut
+    // 3 raz
+    // 4 pri
+    // 5 vpri
+    // 6 prv
+    // 7 dru
+    // 8 tre
+    // 9 čet
+    // 10 man
+    // 11
+    // 12 pot
+    // 13 vpot
+    // 14 tav
+    // 15 pet
+    const floorNumber = [
+      -1,
+      -1,
+      0,
+      0,
+      1,
+      1,
+      2,
+      3,
+      4,
+      null,
+      null,
+      null,
+      null,
+      null,
+      5
+    ];
+
+    return floorNumber[firstFloorId - 1] || null;
+  }
+
+  getNumberOfFloorsFromFloorId(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+
+    const floorIds = floorsIdText.split(",");
+    if (floorIds.length === 0) {
+      return null;
+    }
+
+    return floorIds.length;
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = AktidoCrawler;

app/crawler/specificCrawlers/olx.js

@@ -0,0 +1,888 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  HEATING_TYPE,
+  FURNISHING_TYPE,
+  ACCESS_ROAD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const OLX_ENUMS = {
+  OLX_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje",
+    [CRAWLER_AD_TYPE.ONLY_REQUEST]: "&vrsta=samopotraznja"
+  },
+  OLX_AD_CATEGORY: {
+    [AD_CATEGORY.FLAT.id]: "&kategorija=23",
+    [AD_CATEGORY.HOUSE.id]: "&kategorija=24",
+    [AD_CATEGORY.LAND.id]: "&kategorija=29",
+    [AD_CATEGORY.OFFICE.id]: "&kategorija=25",
+    [AD_CATEGORY.APARTMENT.id]: "&kategorija=27",
+    [AD_CATEGORY.GARAGE.id]: "&kategorija=30",
+    [AD_CATEGORY.COTTAGE.id]: "&kategorija=26"
+  },
+  MAX_DETAIL_FIELDS: 30,
+  OLX_PUBLISHED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm",
+  OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
+};
+
+const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
+
+class OlxCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.ignoredUsernames = ignoredUsernames;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords, existingRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            for (const existingRecord of existingRecords) {
+              const { publishedDate, renewedDate } = existingRecord;
+
+              const publishedDateMoment = moment.utc(publishedDate);
+              const renewedDateMoment = moment.utc(renewedDate);
+
+              const stopCrawlingThisCategory = publishedDateMoment.isSame(
+                renewedDateMoment,
+                "minute"
+              );
+
+              if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
+                generatorsToRemove[index] = true;
+                // console.log("\tGenerator ", index + 1, "has no more new ads");
+                break;
+              }
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[OLX] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $("#rezultatipretrage")
+        .find(".listitem")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+      let status = AD_STATUS.STATUS_NORMAL;
+
+      const propertySelectors = {
+        username:
+          "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
+        title: "#naslovartikla",
+        descriptions: ".artikal_detaljniopis_tekst",
+        category:
+          "#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
+      };
+
+      const username = $(propertySelectors.username)
+        .text()
+        .trim();
+      if (this.ignoredUsernames.includes((username || "").toLowerCase())) {
+        return null;
+      }
+
+      const title = $(propertySelectors.title)
+        .text()
+        .trim();
+      const descriptions = $(propertySelectors.descriptions);
+      const category = $(propertySelectors.category)
+        .text()
+        .trim();
+
+      //====== PRICE DETECTION AND EXTRACTION =====
+      let price = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      const urgentPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
+      )
+        .text()
+        .trim();
+
+      if (normalPriceValue && normalPriceValue.length > 0) {
+        price = normalPriceValue;
+        if (
+          $("#pc > p.n")
+            .text()
+            .indexOf("Hitna") !== -1
+        ) {
+          status = AD_STATUS.STATUS_URGENT;
+        } else {
+          status = AD_STATUS.STATUS_NORMAL;
+        }
+      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.split("KM");
+        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
+        if (priceValues.length > 1) {
+          price = priceValues[1].trim();
+          status = AD_STATUS.STATUS_DISCOUNTED;
+        } else {
+          throw { message: "Can't find urgent price" };
+        }
+      } else {
+        throw {
+          message: "Can't find price (it is not normal nor urgent price ?)"
+        };
+      }
+
+      //====== OTHER AD INFORMATION ===============
+      let adType = null;
+      let olxId = null;
+      let numberOfViewsAgency = null;
+
+      let otherInformationDivId;
+      //We need to locate DIV ID where other information are stored
+      for (let possibleId = 10; possibleId <= 20; possibleId++) {
+        const adTypeFieldTitle = $(
+          `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
+        )
+          .text()
+          .trim();
+
+        if (adTypeFieldTitle === "Vrsta oglasa") {
+          otherInformationDivId = possibleId;
+          break;
+        }
+      }
+
+      if (!otherInformationDivId) {
+        throw { message: "Other information DIV could not be found" };
+      }
+
+      const olxIdFieldSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(4)`;
+      const publishedDateValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(3) > div.df2.neanimiraj > time`;
+      const numberOfViewsAgencyValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(6) > div.df2`;
+      const renewedDateFullValueSelector = `#artikal_glavni_div > div.artikal_lijevo > div.op.ob.pop`;
+
+      const publishedDate = $(publishedDateValueSelector)
+        .text()
+        .trim();
+
+      const publishedDateMoment = moment.tz(
+        publishedDate,
+        OLX_ENUMS.OLX_PUBLISHED_DATE_FORMAT,
+        DEFAULT_TIMEZONE
+      );
+
+      if (!publishedDateMoment.isValid()) {
+        throw { message: "Invalid published date ! Check parsing format" };
+      }
+
+      const renewedDate = $(renewedDateFullValueSelector)
+        .data("content")
+        .trim();
+
+      const renewedDateMoment = moment.tz(
+        renewedDate,
+        OLX_ENUMS.OLX_RENEWED_DATE_FORMAT,
+        DEFAULT_TIMEZONE
+      );
+
+      if (!renewedDateMoment) {
+        throw {
+          message:
+            "Invalid renewed date ! Check how parser parsed renewed date text"
+        };
+      }
+
+      adType = $(
+        `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${otherInformationDivId}) > div:nth-child(2) > div.df2`
+      )
+        .text()
+        .trim();
+
+      const parsedCategory = this.getAdCategoryId(category);
+      if (!parsedCategory) {
+        throw { message: `Unknown ad category [${category}]` };
+      }
+
+      const parsedAdType = this.getAdTypeId(adType);
+      if (!parsedAdType) {
+        throw { message: "Unknown ad type" };
+      }
+
+      const olxIdFieldTitle = $(`${olxIdFieldSelector} > div.df1`)
+        .text()
+        .trim();
+      olxId = $(`${olxIdFieldSelector} > div.df2`)
+        .text()
+        .trim();
+      numberOfViewsAgency = parseInt(
+        $(numberOfViewsAgencyValueSelector)
+          .text()
+          .trim()
+      );
+
+      if (olxIdFieldTitle !== "OLX ID") {
+        throw { message: "Cannot find correct OLX ID" };
+      }
+      //===========================================
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area,
+        gardenSize,
+        numberOfRooms = null,
+        numberOfFloors = null,
+        floor = null,
+        accessRoadType = null,
+        heatingType = null,
+        furnishingType = null,
+        balcony = null,
+        newBuilding = null,
+        elevator = null,
+        water = null,
+        electricity = null,
+        drainageSystem = null,
+        registeredInZkBooks = null,
+        recentlyAdapted = null,
+        parking = null,
+        garage = null,
+        gas = null,
+        antiTheftDoor = null,
+        airCondition = null,
+        phoneConnection = null,
+        cableTV = null,
+        internet = null,
+        basementAttic = null,
+        storeRoom = null,
+        videoSurveillance = null,
+        alarm = null,
+        suitableForStudents = null,
+        includingBills = null,
+        animalsAllowed = null,
+        pool = null,
+        urbanPlanPermit = null,
+        buildingPermit = null,
+        utilityConnection = null,
+        distanceToRiver = null;
+
+      let fieldIndex = 1;
+      do {
+        const fieldSelector = `#dodatnapolja1 > div:nth-child(${fieldIndex})`;
+        const fieldTitleSelector = `${fieldSelector} > div.df1`;
+        const fieldValueSelector = `${fieldSelector} > div.df2`;
+
+        const fieldTitle = $(fieldTitleSelector)
+          .text()
+          .trim()
+          .toLowerCase();
+        const fieldValue = $(fieldValueSelector)
+          .text()
+          .trim()
+          .toLowerCase();
+
+        switch (fieldTitle) {
+          case "kvadrata":
+            area = fieldValue;
+            break;
+          case "okućnica (kvadratura)":
+            gardenSize = fieldValue;
+            break;
+          case "broj soba":
+            numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
+            break;
+          case "broj prostorija":
+            numberOfRooms = this.parseNumberOfRooms(fieldValue, parsedCategory);
+            break;
+          case "broj spratova":
+            numberOfFloors = this.parseNumberOfFloors(
+              fieldValue,
+              parsedCategory
+            );
+            break;
+          case "sprat":
+            floor = this.parseFloorNumber(fieldValue, parsedCategory);
+            break;
+          case "vrsta grijanja":
+            heatingType = this.getHeatingTypeId(fieldValue);
+            break;
+          case "namješten?":
+            furnishingType = this.getFurnishingTypeId(fieldValue);
+            break;
+          case "namješten":
+            furnishingType = FURNISHING_TYPE.FURNISHED.id;
+            break;
+          case "namještena":
+            furnishingType = FURNISHING_TYPE.FURNISHED.id;
+            break;
+          case "voda":
+            water = true;
+            break;
+          case "struja":
+            electricity = true;
+            break;
+          case "kanalizacija":
+            drainageSystem = fieldValue !== "nema";
+            break;
+          case "godina izgradnje":
+            newBuilding = newBuilding || fieldValue === "novogradnja";
+            break;
+          case "kućni ljubimci":
+            animalsAllowed = fieldValue === "da";
+            break;
+          case "uknjiženo / zk":
+            registeredInZkBooks = true;
+            break;
+          case "uknjiženo (zk)":
+            registeredInZkBooks = true;
+            break;
+          case "novogradnja":
+            newBuilding = true;
+            break;
+          case "nedavno adaptiran":
+            recentlyAdapted = true;
+            break;
+          case "nedavno adaptirana":
+            recentlyAdapted = true;
+            break;
+          case "balkon":
+            balcony = true;
+            break;
+          case "lift":
+            elevator = true;
+            break;
+          case "parking":
+            parking = true;
+            break;
+          case "garaža":
+            garage = true;
+            break;
+          case "plin":
+            gas = true;
+            break;
+          case "blindirana vrata":
+            antiTheftDoor = true;
+            break;
+          case "klima":
+            airCondition = true;
+            break;
+          case "telefonski priključak":
+            phoneConnection = true;
+            break;
+          case "kablovska tv":
+            cableTV = true;
+            break;
+          case "internet":
+            internet = true;
+            break;
+          case "podrum/tavan":
+            basementAttic = true;
+            break;
+          case "ostava/špajz":
+            storeRoom = true;
+            break;
+          case "video nadzor":
+            videoSurveillance = true;
+            break;
+          case "alarm":
+            alarm = true;
+            break;
+          case "za studente":
+            suitableForStudents = true;
+            break;
+          case "uključen trošak režija":
+            includingBills = true;
+            break;
+          case "građevinska dozvola":
+            buildingPermit = true;
+            break;
+          case "komunalni priključak":
+            utilityConnection = true;
+            break;
+          case "urbanistička dozvola":
+            urbanPlanPermit = true;
+            break;
+          case "udaljenost od rijeke (m)":
+            distanceToRiver = parseInt(fieldValue) || null;
+            break;
+          case "prilaz":
+            accessRoadType = this.getAccessRoadTypeId(fieldValue);
+            break;
+          case "bazen":
+            pool = true;
+            break;
+          case "iznajmljeno":
+            status = AD_STATUS.STATUS_RENTED;
+            break;
+          default:
+            // console.log(fieldTitle, " = ", fieldValue);
+            break;
+        }
+
+        if (++fieldIndex === OLX_ENUMS.MAX_DETAIL_FIELDS || fieldTitle === "") {
+          break;
+        }
+      } while (true);
+      //===========================================
+
+      //=========================================
+      const parsedArea = this.parseArea(area) || null;
+      const parsedGardenSize = this.parseArea(gardenSize) || null;
+      const parsedPrice = this.parsePrice(price) || null;
+
+      const latLngRegex = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
+      const locationLatLngMatches = latLngRegex.exec(body);
+
+      let locationLat = null;
+      let locationLong = null;
+      if (locationLatLngMatches && locationLatLngMatches.length >= 3) {
+        locationLat = parseFloat(locationLatLngMatches[1]) || null;
+        locationLong = parseFloat(locationLatLngMatches[2]) || null;
+      }
+
+      if (
+        title.indexOf("[PRODANO]") !== -1 ||
+        title.indexOf("[ZAVRŠENO]") !== -1
+      ) {
+        status = AD_STATUS.STATUS_SOLD;
+      }
+
+      const data = {
+        url,
+        agencyObjectId: olxId,
+        originAgencyName: AD_AGENCY.OLX,
+        realEstateType: parsedCategory,
+        adType: parsedAdType,
+        title,
+        price: parsedPrice,
+        area: parsedArea,
+        gardenSize: parsedGardenSize,
+        shortDescription: descriptions
+          .first()
+          .text()
+          .trim(),
+        longDescription: descriptions
+          .last()
+          .text()
+          .trim(),
+        streetNumber: 0,
+        streetName: "",
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat,
+        locationLong,
+        adStatus: status,
+        publishedDate: publishedDateMoment.toISOString(),
+        renewedDate: renewedDateMoment.toISOString(),
+        numberOfRooms,
+        numberOfFloors,
+        floor,
+        accessRoadType,
+        heatingType,
+        furnishingType,
+        balcony,
+        newBuilding,
+        elevator,
+        water,
+        electricity,
+        drainageSystem,
+        registeredInZkBooks,
+        recentlyAdapted,
+        parking,
+        garage,
+        gas,
+        antiTheftDoor,
+        airCondition,
+        phoneConnection,
+        cableTV,
+        internet,
+        basementAttic,
+        storeRoom,
+        videoSurveillance,
+        alarm,
+        suitableForStudents,
+        includingBills,
+        animalsAllowed,
+        pool,
+        urbanPlanPermit,
+        buildingPermit,
+        utilityConnection,
+        distanceToRiver,
+        numberOfViewsAgency
+      };
+
+      return data;
+    } catch (e) {
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "Stanovi":
+        return AD_CATEGORY.FLAT.id;
+      case "Zemljišta":
+        return AD_CATEGORY.LAND.id;
+      case "Kuće":
+        return AD_CATEGORY.HOUSE.id;
+      case "Poslovni prostori":
+        return AD_CATEGORY.OFFICE.id;
+      case "Apartmani":
+        return AD_CATEGORY.APARTMENT.id;
+      case "Garaže":
+        return AD_CATEGORY.GARAGE.id;
+      case "Vikendice":
+        return AD_CATEGORY.COTTAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "Prodaja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "Izdavanje":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      case "Potražnja":
+        return AD_TYPE.AD_TYPE_REQUEST.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  getHeatingTypeId(heatingTypeText) {
+    switch (heatingTypeText) {
+      case "struja":
+        return HEATING_TYPE.ELECTRICITY.id;
+      case "plin":
+        return HEATING_TYPE.GAS.id;
+      case "drva":
+        return HEATING_TYPE.WOOD.id;
+      case "centralno (gradsko)":
+        return HEATING_TYPE.CENTRAL_CITY.id;
+      case "centralno (kotlovnica)":
+        return HEATING_TYPE.CENTRAL_BOILER.id;
+      case "centralno (plin)":
+        return HEATING_TYPE.CENTRAL_GAS.id;
+      case "nije uvedeno":
+        return HEATING_TYPE.NO_HEATING.id;
+      case "ostalo":
+        return HEATING_TYPE.OTHER.id;
+      case "drugo":
+        return HEATING_TYPE.OTHER.id;
+      default:
+        console.log("grijanje = NEPOZNATO [", heatingTypeText, "]");
+        return null;
+    }
+  }
+
+  getFurnishingTypeId(furnishingTypeText) {
+    switch (furnishingTypeText) {
+      case "namješten":
+        return FURNISHING_TYPE.FURNISHED.id;
+      case "polunamješten":
+        return FURNISHING_TYPE.HALF_FURNISHED.id;
+      case "nenamješten":
+        return FURNISHING_TYPE.NOT_FURNISHED.id;
+      case "":
+        return FURNISHING_TYPE.FURNISHED.id;
+      default:
+        console.log("namješten = NEPOZNATO [", furnishingTypeText, "]");
+        return null;
+    }
+  }
+
+  getAccessRoadTypeId(accessRoadTypeText) {
+    switch (accessRoadTypeText) {
+      case "asfalt":
+        return ACCESS_ROAD_TYPE.ASPHALT.id;
+      case "beton":
+        return ACCESS_ROAD_TYPE.CONCRETE.id;
+      case "makadam":
+        return ACCESS_ROAD_TYPE.MACADAM.id;
+      case "ostalo":
+        return ACCESS_ROAD_TYPE.OTHER.id;
+      default:
+        console.log("pristup = NEPOZNATO [", accessRoadTypeText, "]");
+        return null;
+    }
+  }
+
+  parseArea(areaText) {
+    if (!areaText) {
+      return NaN;
+    }
+    const removeDotsExceptLastOneRegex = /[.](?=.*[.])/g;
+    const textWithOnlyOneDecimalDot = areaText
+      .replace(",", ".")
+      .replace(removeDotsExceptLastOneRegex, "");
+
+    return parseFloat(textWithOnlyOneDecimalDot);
+  }
+
+  parsePrice(priceText) {
+    if (!priceText) {
+      return NaN;
+    }
+    const formattedPriceText = priceText.replace(".", "").replace(",", ".");
+    return parseFloat(formattedPriceText);
+  }
+
+  parseNumberOfRooms(numberOfRoomsText, categoryId) {
+    if (categoryId === AD_CATEGORY.FLAT.id) {
+      switch (numberOfRoomsText) {
+        case "garsonjera":
+          return 0;
+        case "jednosoban (1)":
+          return 1;
+        case "jednoiposoban (1.5)":
+          return 1.5;
+        case "dvosoban (2)":
+          return 2;
+        case "trosoban (3)":
+          return 3;
+        case "četverosoban (4)":
+          return 4;
+        case "petosoban i više":
+          return 5;
+        default:
+          console.log(
+            "broj soba [stan] = NEPOZNATO [",
+            numberOfRoomsText,
+            ", ",
+            categoryId,
+            "]"
+          );
+          return null;
+      }
+    }
+
+    if (
+      categoryId === AD_CATEGORY.HOUSE.id ||
+      categoryId === AD_CATEGORY.COTTAGE.id ||
+      categoryId === AD_CATEGORY.APARTMENT.id ||
+      categoryId === AD_CATEGORY.OFFICE.id
+    ) {
+      return parseInt(numberOfRoomsText) || null;
+    }
+
+    console.log("broj soba = NEPOZNATO [", numberOfRoomsText, "]");
+    return null;
+  }
+
+  parseNumberOfFloors(numberOfFloorsText, categoryId) {
+    if (
+      categoryId === AD_CATEGORY.HOUSE.id ||
+      categoryId === AD_CATEGORY.COTTAGE.id
+    ) {
+      return parseInt(numberOfFloorsText) || null;
+    }
+
+    if (categoryId === AD_CATEGORY.OFFICE.id) {
+      if (
+        numberOfFloorsText === "suteren" ||
+        numberOfFloorsText === "prizemlje"
+      ) {
+        return 0;
+      }
+      if (numberOfFloorsText === "6+") {
+        return 7;
+      }
+      return parseInt(numberOfFloorsText) || null;
+    }
+
+    console.log("broj spratova = NEPOZNATO [", numberOfFloorsText, "]");
+    return null;
+  }
+
+  parseFloorNumber(floorText, categoryId) {
+    if (
+      categoryId === AD_CATEGORY.FLAT.id ||
+      categoryId === AD_CATEGORY.APARTMENT.id
+    ) {
+      if (
+        floorText === "suteren" ||
+        floorText === "prizemlje" ||
+        floorText === "visoko prizemlje"
+      ) {
+        return 0;
+      }
+      return parseInt(floorText) || null;
+    }
+
+    if (categoryId === AD_CATEGORY.OFFICE.id) {
+      if (floorText === "zaseban objekat") {
+        return null;
+      }
+      if (floorText === "prizemlje" || floorText === "visoko prizemlje") {
+        return 0;
+      }
+      return parseInt(floorText) || null;
+    }
+
+    console.log("sprat = NEPOZNATO [", floorText, "]");
+    return null;
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = OlxCrawler;

app/crawler/specificCrawlers/prostor.js

@@ -0,0 +1,636 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const moment = require("moment-timezone");
+const FormData = require("form-data");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  FURNISHING_TYPE,
+  HEATING_TYPE
+} = require("../../common/enums");
+
+const {
+  PRINT_CRAWLER_DEBUG,
+  DEFAULT_TIMEZONE,
+  PROSTOR_LOGIN
+} = require("../../config/appConfig");
+const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
+
+const PROSTOR_ENUMS = {
+  PROSTOR_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&action=0",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&action=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&action=2"
+  },
+  PROSTOR_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "&type=7",
+    [AD_CATEGORY.HOUSE.id]: "&type=8",
+    [AD_CATEGORY.LAND.id]: "&type=10",
+    [AD_CATEGORY.OFFICE.id]: "&type=9",
+    [AD_CATEGORY.APARTMENT.id]: "&type=11",
+    [AD_CATEGORY.GARAGE.id]: "&type=14"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  PROSTOR_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  PROSTOR_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+class ProstorCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://prostor.ba/pretraga";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+    //We need session cookie to use login privileges
+    const prostorCookie = await this.getCookies();
+    //New tag to check if crawler loged in
+    const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
+    const newRealEstates = [];
+    //Crawl only if login was successful
+    if (crawlAdCategories && login) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !PROSTOR_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory, prostorCookie) {
+    const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`;
+      const listOfAllRealEstates = await this.extractRealEstates(
+        urlPageToCrawl,
+        prostorCookie
+      );
+
+      let elementToStartIndexFrom = 0;
+      while (true) {
+        const realEstatesForSinglePage = listOfAllRealEstates.slice(
+          elementToStartIndexFrom,
+          elementToStartIndexFrom + this.maxResultsPerPage
+        );
+
+        if (realEstatesForSinglePage.length > 0) {
+          elementToStartIndexFrom += realEstatesForSinglePage.length;
+
+          const singlePageResults = await this.indexSinglePage(
+            realEstatesForSinglePage,
+            prostorCookie
+          );
+
+          const filteredSinglePageResults = singlePageResults.filter(
+            singleResult => !!singleResult
+          );
+
+          if (
+            Array.isArray(filteredSinglePageResults) &&
+            filteredSinglePageResults.length > 0
+          ) {
+            yield filteredSinglePageResults;
+          } else {
+            return undefined;
+          }
+        } else {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(realEstatesList, prostorCookie) {
+    const asyncActions = [];
+    for (const realEstate of realEstatesList) {
+      asyncActions.push(this.scrapeAd(realEstate, prostorCookie));
+    }
+
+    try {
+      return await Promise.all(asyncActions);
+    } catch (e) {
+      console.log(
+        "[PROSTOR] Error crawling ads : ",
+        e.message || "UNKNOWN ERROR"
+      );
+      return [];
+    }
+  }
+
+  async scrapeAd(realEstate, prostorCookie) {
+    const { lat, lng, property_name, price, size, link, status } = realEstate;
+
+    //Status information is given already in realestate list
+    const adStatus = ProstorCrawler.getStatusId(status);
+
+    const url = `https://prostor.ba${link}`;
+
+    // console.log("[PROSTOR] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url, {
+        headers: { Cookie: prostorCookie }
+      });
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      // link contains part of the URL in the format of : /prodaja/stan/stup/9556
+      // general form is : /actionType/realEstateType/location/realEstateID
+      // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
+
+      const linkParts = link.split("/");
+
+      const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
+      const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
+      const prostorId = linkParts[4];
+
+      if (!adType || !realEstateType || !prostorId) {
+        return null;
+      }
+
+      const allDataSelector =
+        "body > div > div.container-fluid > div > div.column-right > table > tbody";
+
+      const realEstateProperties = {};
+
+      $(allDataSelector)
+        .find("p")
+        .each((i, element) => {
+          const propertyElement = $(element)
+            .text()
+            .split(":")
+            .map(text => text.trim().toLowerCase());
+
+          const propertyTitle = propertyElement[0];
+          realEstateProperties[propertyTitle] = propertyElement[1];
+        });
+
+      $(allDataSelector)
+        .find("div.mb-2")
+        .each((i, element) => {
+          const propertyElement = $(element)
+            .text()
+            .trim()
+            .toLowerCase();
+
+          realEstateProperties[propertyElement] = true;
+        });
+
+      if (JSON.stringify(realEstateProperties) === JSON.stringify({})) {
+        return null;
+      }
+
+      let numberOfRooms =
+          parseFloat(realEstateProperties["broj soba"]) +
+            parseFloat(realEstateProperties["broj spavaćih soba"]) || null,
+        numberOfFloors = null,
+        floor = null,
+        accessRoadType = null,
+        heatingType = ProstorCrawler.getHeatingTypeId(realEstateProperties),
+        furnishingType = null,
+        balcony =
+          realEstateProperties["balkon"] ||
+          realEstateProperties["terasa"] ||
+          realEstateProperties["lođa"] ||
+          null,
+        newBuilding = linkParts[1] === "novogradnja",
+        elevator = realEstateProperties["lift"] || null,
+        water = realEstateProperties["voda"] || null,
+        electricity = realEstateProperties["električna energija"] || null,
+        drainageSystem = realEstateProperties["kanalizacija"] || null,
+        registeredInZkBooks = null,
+        recentlyAdapted = null,
+        parking = realEstateProperties["parking"] || null,
+        garage = realEstateProperties["garaža"] || null,
+        gas = realEstateProperties["plin"] || null,
+        antiTheftDoor = realEstateProperties["blindo vrata"] || null,
+        airCondition = realEstateProperties["klima"] || null,
+        phoneConnection = realEstateProperties["telefon"] || null,
+        cableTV = realEstateProperties["kablovksa tv"] || null,
+        internet =
+          realEstateProperties["internet"] ||
+          realEstateProperties["adsl"] ||
+          null,
+        basementAttic = realEstateProperties["podrum"] || null,
+        storeRoom = realEstateProperties["ostava"] || null,
+        videoSurveillance = realEstateProperties["video nadzor"],
+        alarm = realEstateProperties["alarm"] || null,
+        suitableForStudents = null,
+        includingBills = null,
+        animalsAllowed = null,
+        pool = realEstateProperties["bazen"] || null,
+        urbanPlanPermit = null,
+        buildingPermit = null,
+        utilityConnection = null,
+        distanceToRiver = null,
+        numberOfViewsAgency = null;
+
+      // Floor versions (there are possibly more versions) :
+      // Sprat: 3/3
+      // Sprat: 1 - 2/2
+      // Sprat: Pr - 7/7
+      // Sprat: -2/0
+      // If there are two parts, that represents more real estates are sold
+      // numberOfFloors is contained in second part, after / sign
+
+      const floorsArray = realEstateProperties["sprat"].split(" - ");
+      let floorText = "";
+      if (floorsArray.length === 1) {
+        const floorDescription = floorsArray[0].split("/");
+        numberOfFloors = parseInt(floorDescription[1]) || null;
+        floorText = floorDescription[0];
+        floor = Math.round(parseFloat(floorText));
+      } else if (floorsArray.length === 2) {
+        const floorDescription = floorsArray[1].split("/");
+        numberOfFloors = parseInt(floorDescription[1]) || null;
+        floorText = floorsArray[0];
+        floor = Math.round(parseFloat(floorText));
+      } else {
+        // This is something strange
+      }
+
+      if (isNaN(floor)) {
+        // It was textual representation of floor, like "Pr", "Su" or similar
+        switch (floorText) {
+          case "pr":
+            floor = 0;
+            break;
+          case "su":
+            floor = -1;
+            break;
+          default:
+            console.log(
+              "[PROSTOR] Unknown textual representation of floor : ",
+              floorText
+            );
+            floor = null;
+        }
+      }
+
+      if (realEstateProperties["namješteno"]) {
+        furnishingType = FURNISHING_TYPE.FURNISHED.id;
+      } else if (realEstateProperties["polunamješteno"]) {
+        furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
+      } else {
+        furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
+      }
+
+      const title = property_name;
+      const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
+      const parsedArea = parseFloat(size);
+      const gardenSize = null;
+      const longDescription = null;
+
+      const data = {
+        url,
+        agencyObjectId: prostorId,
+        originAgencyName: AD_AGENCY.PROSTOR,
+        realEstateType,
+        adType,
+        title,
+        price: parsedPrice,
+        area: parsedArea,
+        gardenSize,
+        shortDescription: "",
+        longDescription: longDescription,
+        streetNumber: 0,
+        streetName: realEstateProperties["adresa"],
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat: lat,
+        locationLong: lng,
+        adStatus,
+        numberOfRooms,
+        numberOfFloors,
+        floor,
+        accessRoadType,
+        heatingType,
+        furnishingType,
+        balcony,
+        newBuilding,
+        elevator,
+        water,
+        electricity,
+        drainageSystem,
+        registeredInZkBooks,
+        recentlyAdapted,
+        parking,
+        garage,
+        gas,
+        antiTheftDoor,
+        airCondition,
+        phoneConnection,
+        cableTV,
+        internet,
+        basementAttic,
+        storeRoom,
+        videoSurveillance,
+        alarm,
+        suitableForStudents,
+        includingBills,
+        animalsAllowed,
+        pool,
+        urbanPlanPermit,
+        buildingPermit,
+        utilityConnection,
+        distanceToRiver,
+        numberOfViewsAgency
+      };
+
+      return data;
+    } catch (e) {
+      console.error(
+        "[PROSTOR] Exception caught: " + e.message,
+        "\r\nURL:",
+        url
+      );
+      return null;
+    }
+  }
+
+  async extractRealEstates(url, prostorCookie) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[PROSTOR] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url, {
+        headers: { Cookie: prostorCookie }
+      });
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      const scriptElement = $(
+        "body > div > div.container-fluid > script:nth-child(7)"
+      );
+
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        const scriptData = scriptElement[0].children[0].data;
+
+        try {
+          // script element data contains JS code and we need to extract only data for realEstates
+          // data string starts with : var map; var markers = [{"r ...
+          // so we remove first 23 characters
+          //
+          // real estate JSON data ends with      ...}, ]; map = new...
+          // so we need to find index of that substring to know where to stop
+          // we will NOT include trailing comma because it breaks JSON parse, so we have to close ] bracket manually
+
+          const jsonEndIndex = scriptData.indexOf(", ]; map = new");
+          if (jsonEndIndex > -1) {
+            const jsonData = scriptData.substring(23, jsonEndIndex) + "]";
+            const realEstates = JSON.parse(jsonData);
+
+            // const transformedRealEstates = [];
+            //
+            // for (const realEstate of realEstates) {
+            //   const transformedRealEstate = ProstorCrawler.transformRealEstateData(
+            //     realEstate
+            //   );
+            //   if (transformedRealEstate) {
+            //     transformedRealEstates.push(transformedRealEstate);
+            //   }
+            // }
+            //
+            // return transformedRealEstates;
+            return realEstates;
+          } else {
+            throw {
+              message: "Something is wrong with JSON data or data is moved"
+            };
+          }
+        } catch (e) {
+          console.log(e);
+          throw e;
+        }
+      }
+    } catch (e) {
+      console.error(
+        "[PROSTOR] Exception caught:",
+        e.message || "UNKNOWN MESSAGE"
+      );
+      return [];
+    }
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  static getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "stan":
+        return AD_CATEGORY.FLAT.id;
+      case "kuca":
+        return AD_CATEGORY.HOUSE.id;
+      case "apartman":
+        return AD_CATEGORY.APARTMENT.id;
+      case "poslovni-prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "garaza":
+        return AD_CATEGORY.GARAGE.id;
+      case "zemljiste":
+        return AD_CATEGORY.LAND.id;
+      default:
+        return undefined;
+    }
+  }
+
+  static getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "prodaja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "najam":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      case "novogradnja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  static getHeatingTypeId(realEstateProperties) {
+    const realEstatePropertiesKeys = Object.keys(realEstateProperties);
+    for (const property of realEstatePropertiesKeys) {
+      switch (property) {
+        case "centralno toplane":
+          return HEATING_TYPE.CENTRAL_CITY.id;
+        case "etažno plinsko":
+          return HEATING_TYPE.CENTRAL_GAS.id;
+        case "termo blok":
+        case "podno grijanje":
+          return HEATING_TYPE.OTHER.id;
+        case "etažno električno":
+        case "konvektori":
+          return HEATING_TYPE.ELECTRICITY.id;
+        case "plinske peći":
+          return HEATING_TYPE.GAS.id;
+        case "vlastita kotlovnica":
+          return HEATING_TYPE.CENTRAL_BOILER.id;
+        case "toplotna pumpa":
+          return HEATING_TYPE.HEAT_PUMP.id;
+        case "kamin":
+          return HEATING_TYPE.WOOD.id;
+        default:
+        //console.log("[PROSTOR] Nepoznato >>>  [", property, "]");
+      }
+    }
+  }
+
+  static getStatusId(statusText) {
+    switch (statusText) {
+      case "":
+        return AD_STATUS.STATUS_NORMAL;
+      case "Rezervisano":
+        return AD_STATUS.STATUS_RESERVED;
+      case "Prodano":
+        return AD_STATUS.STATUS_SOLD;
+      case "Iznajmljeno":
+        return AD_STATUS.STATUS_RENTED;
+      case "VIP ponuda":
+        return AD_STATUS.STATUS_VIP;
+      default:
+        console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]");
+        return AD_STATUS.STATUS_NORMAL;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+  async loginForScraping(PROSTOR_LOGIN, prostorCookie) {
+    let formData = new FormData();
+    formData.append("email", PROSTOR_LOGIN.EMAIL);
+    formData.append("password", PROSTOR_LOGIN.PASSWORD);
+
+    return fetch("https://prostor.ba/moj-prostor/prijava", {
+      method: "POST",
+      body: formData,
+      headers: { Cookie: prostorCookie }
+    })
+      .then(page => {
+        return page.text();
+      })
+      .then(resp => {
+        const $ = cheerio.load(resp);
+        if (
+          $("h1")
+            .text()
+            .indexOf("Dobrodošli") !== -1
+        ) {
+          console.log("[PROSTOR]: Crawler loged in!");
+          return true;
+        } else {
+          console.log("[PROSTOR]: Crawler login failed - wrong credentials!");
+          return false;
+        }
+      })
+      .catch(err => {
+        console.log("[PROSTOR]: Crawler login error ", err);
+      });
+  }
+  async getCookies() {
+    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
+      headers: { Cookie: "" }
+    });
+    const raw = getResponse.headers.raw()["set-cookie"];
+    const cookie = raw
+      .map(datastring => {
+        const data = datastring.split(";");
+        const cookieData = data[0];
+        return cookieData;
+      })
+      .join(";");
+    return cookie;
+  }
+}
+
+module.exports = ProstorCrawler;

app/crawler/specificCrawlers/rental.js

@@ -0,0 +1,842 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  HEATING_TYPE,
+  ACCESS_ROAD_TYPE,
+  FURNISHING_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const RENTAL_ENUMS = {
+  RENTAL_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  RENTAL_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental");
+
+class RentalCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !RENTAL_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[RENTAL] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".pull-right")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[RENTAL] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[RENTAL] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        let adStatus = AD_STATUS.STATUS_NORMAL;
+        const rentalId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromRentalId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromRentalActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const descriptionIds = extractedData["re_descriptions_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(descriptionIds)) {
+          throw {
+            message:
+              'Expected array od descriptions but "re_descriptions_id" not found !'
+          };
+        }
+
+        const spaceIds = extractedData["re_spaces_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(spaceIds)) {
+          throw {
+            message: 'Expected array od spaces but "re_spaces_id" not found !'
+          };
+        }
+
+        const infrastructureIds = extractedData["re_infrastructure_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(infrastructureIds)) {
+          throw {
+            message:
+              'Expected array od infrastructures but "re_infrastructure_id" not found !'
+          };
+        }
+
+        const floorNoIds = extractedData["re_floorNO_id"]
+          .split(",")
+          .map(stringNumber => parseInt(stringNumber));
+
+        if (!Array.isArray(floorNoIds)) {
+          throw {
+            message:
+              'Expected array od infrastructures but "re_floorNO_id" not found !'
+          };
+        }
+
+        const numberOfViewsAgencySelector = $(
+          "body > div > div.container > div.row.content-top > div.col-xs-12.col-sm-12.col-md-9 > div > div.box-viewcount"
+        );
+
+        // number of views is written as : "Broj pregledavanja: NNN"
+        const numberOfViewsAgencyFullText = numberOfViewsAgencySelector
+          .text()
+          .trim();
+
+        const numberOfViewsAgencyParts = numberOfViewsAgencyFullText.split(":");
+
+        const realEstatePropertiesFromDescriptions = this.getPropertiesFromDescriptions(
+          descriptionIds
+        );
+        const realEstatePropertiesFromSpaces = this.getPropertiesFromSpaces(
+          spaceIds
+        );
+
+        const realEstatePropertiesFromInfrastructure = this.getPropertiesFromInfrastructure(
+          infrastructureIds
+        );
+
+        if (extractedData["adm_realEstates_discount"] === "1") {
+          adStatus = AD_STATUS.STATUS_DISCOUNTED;
+        }
+
+        let numberOfRooms =
+            parseInt(extractedData["re_realEstates_roomsNO"]) +
+              parseInt(extractedData["re_realEstates_bedNO"]) || null,
+          numberOfFloors =
+            parseInt(extractedData["re_realEstates_floorsNO"]) ||
+            this.getNumberOfFloorsFromFloorId(extractedData["re_floorNO_id"]),
+          floor =
+            parseInt(extractedData["re_realEstates_floorNO"]) ||
+            this.getFloorNumberFromFloorId(extractedData["re_floorNO_id"]),
+          accessRoadType = realEstatePropertiesFromDescriptions.accessRoadType,
+          heatingType =
+            this.getHeatingTypeId(extractedData["re_heating_id"]) || null,
+          furnishingType = realEstatePropertiesFromDescriptions.furnishingType,
+          balcony =
+            realEstatePropertiesFromDescriptions.balcony ||
+            realEstatePropertiesFromSpaces.balcony,
+          newBuilding = extractedData["op_realEstates_newBuilding"]
+            ? extractedData["op_realEstates_newBuilding"] === "1"
+            : null,
+          elevator = realEstatePropertiesFromDescriptions.elevator,
+          water =
+            realEstatePropertiesFromDescriptions.water ||
+            realEstatePropertiesFromInfrastructure.water,
+          electricity =
+            realEstatePropertiesFromDescriptions.electricity ||
+            realEstatePropertiesFromInfrastructure.electricity,
+          drainageSystem =
+            realEstatePropertiesFromInfrastructure.drainageSystem,
+          registeredInZkBooks =
+            extractedData["op_realEstates_ownerPermit"] === 1 || null,
+          recentlyAdapted = null,
+          parking =
+            realEstatePropertiesFromDescriptions.parking ||
+            realEstatePropertiesFromSpaces.parking,
+          garage = realEstatePropertiesFromSpaces.garage,
+          gas = realEstatePropertiesFromInfrastructure.gas,
+          antiTheftDoor = realEstatePropertiesFromDescriptions.antiTheftDoor,
+          airCondition = realEstatePropertiesFromDescriptions.airCondition,
+          phoneConnection =
+            realEstatePropertiesFromInfrastructure.phoneConnection,
+          cableTV = realEstatePropertiesFromInfrastructure.cableTV,
+          internet = realEstatePropertiesFromInfrastructure.internet,
+          basementAttic =
+            realEstatePropertiesFromSpaces.basementAttic ||
+            this.checkBasemAtticFromFloors(extractedData["re_floorNO_id"]),
+          storeRoom = realEstatePropertiesFromSpaces.storeRoom,
+          videoSurveillance =
+            realEstatePropertiesFromDescriptions.videoSurveillance ||
+            realEstatePropertiesFromInfrastructure.videoSurveillance,
+          alarm = realEstatePropertiesFromDescriptions.alarm,
+          suitableForStudents = null,
+          includingBills =
+            extractedData["op_realEstates_utilitiesIncluded"] === "1" || null,
+          animalsAllowed = null,
+          pool = realEstatePropertiesFromDescriptions.pool,
+          urbanPlanPermit =
+            extractedData["op_realEstates_locationPermit"] === "1" ||
+            realEstatePropertiesFromDescriptions.urbanPlanPermit,
+          buildingPermit =
+            extractedData["op_realEstates_buildingPermit"] === "1" || null,
+          utilityConnection =
+            realEstatePropertiesFromDescriptions.utilityConnection,
+          distanceToRiver = null,
+          numberOfViewsAgency =
+            numberOfViewsAgencyParts.length > 1
+              ? parseInt(numberOfViewsAgencyParts[1])
+              : null;
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const data = {
+          url,
+          agencyObjectId: rentalId,
+          originAgencyName: AD_AGENCY.RENTAL,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString(),
+          numberOfRooms,
+          numberOfFloors,
+          floor,
+          accessRoadType,
+          heatingType,
+          furnishingType,
+          balcony,
+          newBuilding,
+          elevator,
+          water,
+          electricity,
+          drainageSystem,
+          registeredInZkBooks,
+          recentlyAdapted,
+          parking,
+          garage,
+          gas,
+          antiTheftDoor,
+          airCondition,
+          phoneConnection,
+          cableTV,
+          internet,
+          basementAttic,
+          storeRoom,
+          videoSurveillance,
+          alarm,
+          suitableForStudents,
+          includingBills,
+          animalsAllowed,
+          pool,
+          urbanPlanPermit,
+          buildingPermit,
+          utilityConnection,
+          distanceToRiver,
+          numberOfViewsAgency
+        };
+
+        return data;
+      } else {
+        console.log("[RENTAL] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromRentalId(rentalCategoryId) {
+    switch (rentalCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromRentalActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  getPropertiesFromDescriptions(descriptionIds) {
+    const result = {
+      accessRoadType: null,
+      furnishingType: null,
+      balcony: null,
+      elevator: null,
+      parking: null,
+      antiTheftDoor: null,
+      airCondition: null,
+      videoSurveillance: null,
+      alarm: null,
+      pool: null,
+      urbanPlanPermit: null,
+      utilityConnection: null,
+      water: null,
+      electricity: null
+    };
+
+    for (const descriptionId of descriptionIds) {
+      switch (descriptionId) {
+        case 16:
+          result.furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
+          break;
+        case 17:
+          result.furnishingType = FURNISHING_TYPE.HALF_FURNISHED.id;
+          break;
+        case 1:
+        case 28:
+          result.furnishingType = FURNISHING_TYPE.FURNISHED.id;
+          break;
+        case 14:
+          result.elevator = true;
+          break;
+        case 39:
+          result.electricity = true;
+          break;
+        case 40:
+          result.water = true;
+          break;
+        case 41:
+        case 58:
+          result.accessRoadType = ACCESS_ROAD_TYPE.ASPHALT.id;
+          break;
+        case 26:
+          result.balcony = true;
+          break;
+        case 62:
+          result.parking = true;
+          break;
+        case 3:
+          result.antiTheftDoor = true;
+          break;
+        case 2:
+        case 21:
+          result.airCondition = true;
+          break;
+        case 4:
+          result.alarm = true;
+          break;
+        case 55:
+          result.videoSurveillance = true;
+          break;
+        case 9:
+          result.pool = true;
+          break;
+        case 60:
+          result.urbanPlanPermit = true;
+          break;
+        case 38:
+          result.utilityConnection = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getPropertiesFromSpaces(spaceIds) {
+    const result = {
+      balcony: null,
+      parking: null,
+      garage: null,
+      basementAttic: null,
+      storeRoom: null
+    };
+
+    for (const spaceId of spaceIds) {
+      switch (spaceId) {
+        case 36:
+        case 12:
+          result.parking = true;
+          break;
+        case 1:
+        case 2:
+        case 3:
+          result.balcony = true;
+          break;
+        case 4:
+        case 30:
+          result.garage = true;
+          break;
+        case 9:
+        case 10:
+          result.storeRoom = true;
+          break;
+        case 18:
+        case 34:
+        case 37:
+        case 27:
+          result.basementAttic = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getHeatingTypeId(heatingRentalId) {
+    // heatingRentalId can have multiple values, like: "1, 2, 3", parseInt will take first integer value
+    const heatingId = parseInt(heatingRentalId);
+    switch (heatingId) {
+      case 27:
+      case 16:
+        return HEATING_TYPE.GAS.id;
+      case 4:
+        return HEATING_TYPE.CENTRAL_GAS.id;
+      case 3:
+      case 23:
+      case 6:
+      case 7:
+      case 8:
+      case 9:
+      case 10:
+        return HEATING_TYPE.CENTRAL_BOILER.id;
+      case 2:
+      case 13:
+      case 30:
+      case 17:
+      case 29:
+      case 31:
+        return HEATING_TYPE.ELECTRICITY.id;
+      case 24:
+      case 25:
+      case 12:
+        return HEATING_TYPE.CENTRAL_CITY.id;
+      case 26:
+      case 21:
+      case 20:
+        return HEATING_TYPE.WOOD.id;
+      case 28:
+      case 19:
+        return HEATING_TYPE.HEAT_PUMP.id;
+      case 14:
+      case 32:
+        return HEATING_TYPE.OTHER.id;
+      default:
+        return null;
+    }
+  }
+
+  getPropertiesFromInfrastructure(infrastructureIds) {
+    const result = {
+      electricity: null,
+      water: null,
+      gas: null,
+      drainageSystem: null,
+      phoneConnection: null,
+      internet: null,
+      videoSurveillance: null,
+      cableTV: null
+    };
+
+    for (const infrastructureId of infrastructureIds) {
+      switch (infrastructureId) {
+        case 1:
+          result.electricity = true;
+          break;
+        case 2:
+          result.water = true;
+          break;
+        case 4:
+          result.gas = true;
+          break;
+        case 5:
+          result.drainageSystem = true;
+          break;
+        case 7:
+        case 8:
+          result.phoneConnection = true;
+          break;
+        case 10:
+          result.internet = true;
+          break;
+        case 11:
+          result.cableTV = true;
+          break;
+        case 16:
+        case 17:
+          result.videoSurveillance = true;
+          break;
+      }
+    }
+
+    return result;
+  }
+
+  getFloorNumberFromFloorId(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+    // just extracting floor number from first element
+
+    const floorsId = floorsIdText.split(",");
+    if (floorsId.length === 0) {
+      return null;
+    }
+
+    const firstFloorId = parseInt(floorsId[0]);
+
+    // 1 pod
+    // 2 sut
+    // 3 raz
+    // 4 pri
+    // 5 vpri
+    // 6 prv
+    // 7 dru
+    // 8 tre
+    // 9 čet
+    // 10 man
+    // 11
+    // 12 pot
+    // 13 vpot
+    // 14 tav
+    // 15 pet
+    const floorNumber = [
+      -1,
+      -1,
+      0,
+      0,
+      1,
+      1,
+      2,
+      3,
+      4,
+      null,
+      null,
+      null,
+      null,
+      null,
+      5
+    ];
+
+    return floorNumber[firstFloorId - 1] || null;
+  }
+
+  getNumberOfFloorsFromFloorId(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+
+    const floorIds = floorsIdText.split(",");
+    if (floorIds.length === 0) {
+      return null;
+    }
+    let noOfFloors = floorIds.length;
+    // Floors of 'suteren', 'podrum', 'tavan' and 'potkrovlje' are not counted
+    floorIds.forEach(id => {
+      if (
+        parseInt(id) === 1 ||
+        parseInt(id) === 2 ||
+        parseInt(id) === 12 ||
+        parseInt(id) === 14
+      ) {
+        noOfFloors--;
+      }
+    });
+    return noOfFloors;
+  }
+
+  checkBasemAtticFromFloors(floorsIdText) {
+    // floorIdText can be array of numbers, separated by comma or number
+    const floorIds = floorsIdText.split(",");
+
+    let check = false;
+
+    if (floorIds.length === 0) {
+      check = false;
+    }
+    //If floors 'suteren', 'podrum', 'tavan' and 'potkrovlje' exists then tag for basement-attic is true
+    floorIds.forEach(id => {
+      if (
+        parseInt(id) === 1 ||
+        parseInt(id) === 2 ||
+        parseInt(id) === 12 ||
+        parseInt(id) === 14
+      ) {
+        check = true;
+      }
+    });
+    return check;
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = RentalCrawler;

app/crawler/specificCrawlers/saljic.js

@@ -0,0 +1,630 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const moment = require("moment-timezone");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  FURNISHING_TYPE,
+  HEATING_TYPE
+} = require("../../common/enums");
+
+const {
+  PRINT_CRAWLER_DEBUG,
+  DEFAULT_TIMEZONE
+} = require("../../config/appConfig");
+const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
+
+const SALJIC_ENUMS = {
+  SALJIC_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
+  },
+  SALJIC_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "&input_kategorija=",
+    [AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
+    [AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
+    [AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
+    [AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
+    [AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
+    [AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  }
+};
+
+class SaljicCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+      //
+      //console.log(indexGenerators);
+      //
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !SALJIC_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
+
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
+        const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
+
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[SALJIC] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let adTypesTmp = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const adType = $(elem)
+            .find(".trakica-search-page")
+            .text()
+            .trim();
+          if (adType) {
+            adTypesTmp.push(adType);
+          }
+        });
+
+      //Converting to AD_TYPE
+      const adTypes = adTypesTmp.map(adTypeText => {
+        return this.getAdTypeId(adTypeText);
+      });
+
+      //Converting to absolute URLs
+      const hrefsAbs = hrefs.map(link => {
+        return "https://www.saljicnekretnine.ba" + link;
+      });
+
+      let actualNoOfResults =
+        hrefsAbs.length <= maxResultsPerPage
+          ? hrefsAbs.length
+          : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[SALJIC] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url, adType) {
+    // console.log("[SALJIC] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      // No information for status ex. PRODAN
+      const status = AD_STATUS.STATUS_NORMAL;
+      //Extracting agency ID from url
+      const agencyObjectId = parseInt(url.substring(46, url.length));
+
+      //Extracting main properties
+      const propertySelectors = {
+        title:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
+        price:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
+        streetName:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
+
+        descriptions:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
+        latAndLong:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
+      };
+      const title = $(propertySelectors.title)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+
+      const priceText = $(propertySelectors.price)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+      const price =
+        priceText === "CIJENA NA UPIT"
+          ? null
+          : parseFloat(
+              priceText.substring(8, priceText.length - 3).replace(",", "")
+            );
+
+      const streetName = $(propertySelectors.streetName)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .trim();
+
+      const descriptions = $(propertySelectors.descriptions)
+        .text()
+        .replace(/\"/g, "")
+        .trim();
+
+      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
+      const latText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("marker=") + 7,
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
+      );
+      const longText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
+        latAndLongSrc.length
+      );
+      const locationLat = parseFloat(latText) || null;
+      const locationLong = parseFloat(longText) || null;
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area = null,
+        gardenSize = null,
+        numberOfRooms = null,
+        numberOfFloors = null,
+        floor = null,
+        accessRoadType = null,
+        heatingType = null,
+        furnishingType = null,
+        balcony = null,
+        newBuilding = null,
+        elevator = null,
+        water = null,
+        electricity = null,
+        drainageSystem = null,
+        registeredInZkBooks = null,
+        recentlyAdapted = null,
+        parking = null,
+        garage = null,
+        gas = null,
+        antiTheftDoor = null,
+        airCondition = null,
+        phoneConnection = null,
+        cableTV = null,
+        internet = null,
+        basementAttic = null,
+        storeRoom = null,
+        videoSurveillance = null,
+        alarm = null,
+        suitableForStudents = null,
+        includingBills = null,
+        animalsAllowed = null,
+        pool = null,
+        exchange = null,
+        urbanPlanPermit = null,
+        buildingPermit = null,
+        utilityConnection = null,
+        distanceToRiver = null;
+      let publishedDate = null;
+      let renewedDate = null;
+      let realEstateType;
+      let numberOfViewsAgency = null;
+      let numberOfViewsKivi = null;
+      let streetNumber = 0;
+      let adStatus = status;
+      let shortDescription = descriptions.substring(
+        0,
+        descriptions.indexOf(".")
+      );
+      let longDescription = descriptions;
+      //Extracting data - Glavne karakteristike
+      let mainFieldIndex = 1;
+      do {
+        const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
+
+        const mainField = $(mainFieldSelector)
+          .text()
+          .replace(/[\n\r\t]/gm, "")
+          .trim();
+
+        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldValue = mainField
+          .substring(mainField.indexOf("  "), mainField.length)
+          .trim();
+
+        switch (mainFieldTitle) {
+          case "Površina":
+            area = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Okućnica":
+            gardenSize = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Broj soba":
+            numberOfRooms = parseInt(mainFieldValue);
+            break;
+          case "Broj spratova":
+            numberOfFloors = parseInt(mainFieldValue);
+            break;
+          case "Sprat":
+            floor = parseInt(mainFieldValue);
+            break;
+          case "Godina renoviranja":
+            recentlyAdapted = true;
+            break;
+          case "Broj parking mjesta":
+            parking = true;
+            break;
+          case "Dostupno od":
+            const day = mainFieldValue.substring(0, 2);
+            const month = mainFieldValue.substring(3, 5);
+            const year = mainFieldValue.substring(6, mainFieldValue.length);
+            publishedDate = new Date(`${month}/${day}/${year}`);
+            break;
+          default:
+            break;
+        }
+
+        if (mainFieldTitle === "") {
+          break;
+        }
+        mainFieldIndex++;
+      } while (true);
+
+      //Extracting data - Sadrzaji
+      let additionalFieldIndex = 1;
+      do {
+        const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
+
+        const additionalField = $(additionalFieldSelector)
+          .text()
+          .trim();
+
+        if (additionalFieldIndex === 1) {
+          //Extracting data of real estate type
+          const categoryTmp = additionalField
+            .replace(/[\n\r\t]/gm, "")
+            .substring(
+              additionalField.indexOf("Kategorija") + 10,
+              additionalField.length
+            )
+            .trim();
+          realEstateType = this.getAdCategoryId(categoryTmp);
+        } else {
+          switch (additionalField) {
+            case "Internet":
+              internet = true;
+              break;
+            case "Garaža":
+              garage = true;
+              break;
+            case "Klima":
+              airCondition = true;
+              break;
+            case "Balkon":
+              balcony = true;
+              break;
+            case "Ostava":
+              storeRoom = true;
+              break;
+            case "Podrum":
+              basementAttic = true;
+              break;
+            case "Blindirana vrata":
+              antiTheftDoor = true;
+              break;
+            case "Voda":
+              water = true;
+              break;
+            case "Kablovska":
+              cableTV = true;
+              break;
+            case "Uknjiženo":
+              registeredInZkBooks = true;
+              break;
+            case "Grijanje - centralno":
+              heatingType = HEATING_TYPE.CENTRAL_CITY.id;
+              break;
+            case "Grijanje - plin":
+              heatingType = HEATING_TYPE.GAS.id;
+              break;
+            case "Grijanje - struja":
+              heatingType = HEATING_TYPE.ELECTRICITY.id;
+              break;
+            case "Grijanje":
+              heatingType = HEATING_TYPE.OTHER.id;
+              break;
+            case "Plin":
+              gas = true;
+              break;
+            case "Namješten":
+              furnishingType = FURNISHING_TYPE.FURNISHED.id;
+              break;
+            case "Alarm":
+              alarm = true;
+              break;
+            case "Video nadzor":
+              videoSurveillance = true;
+              break;
+            case "Lift":
+              elevator = true;
+              break;
+            case "Novogradnja":
+              newBuilding = true;
+              break;
+
+            default:
+              break;
+          }
+        }
+
+        if (additionalField === "") {
+          break;
+        }
+        additionalFieldIndex++;
+      } while (true);
+
+      //If no published date it takes current date of crawling
+      if (publishedDate) {
+        renewedDate = new Date();
+      } else {
+        publishedDate = new Date();
+        renewedDate = new Date();
+      }
+
+      const originAgencyName = AD_AGENCY.SALJIC;
+      const locality = "";
+      const municipality = "";
+      const city = "";
+      const region = "";
+      const entity = "";
+      const country = "";
+
+      const data = {
+        url,
+        agencyObjectId,
+        originAgencyName,
+        realEstateType,
+        adType,
+        title,
+        price,
+        area,
+        gardenSize,
+        shortDescription,
+        longDescription,
+        streetNumber,
+        streetName,
+        locality,
+        municipality,
+        city,
+        region,
+        entity,
+        country,
+        locationLat,
+        locationLong,
+        adStatus,
+        publishedDate,
+        renewedDate,
+        numberOfRooms,
+        numberOfFloors,
+        floor,
+        accessRoadType,
+        heatingType,
+        furnishingType,
+        balcony,
+        newBuilding,
+        elevator,
+        water,
+        electricity,
+        drainageSystem,
+        registeredInZkBooks,
+        recentlyAdapted,
+        parking,
+        garage,
+        gas,
+        antiTheftDoor,
+        airCondition,
+        phoneConnection,
+        cableTV,
+        internet,
+        basementAttic,
+        storeRoom,
+        videoSurveillance,
+        alarm,
+        suitableForStudents,
+        includingBills,
+        animalsAllowed,
+        pool,
+        exchange,
+        urbanPlanPermit,
+        buildingPermit,
+        utilityConnection,
+        distanceToRiver,
+        numberOfViewsAgency,
+        numberOfViewsKivi
+      };
+
+      return data;
+    } catch (e) {
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "Stan":
+        return AD_CATEGORY.FLAT.id;
+      case "Građevinsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Industrijsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Poljoprivredno zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Kuća":
+        return AD_CATEGORY.HOUSE.id;
+      case "Poslovni prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "Kancelarije":
+        return AD_CATEGORY.OFFICE.id;
+      case "Apartmani":
+        return AD_CATEGORY.APARTMENT.id;
+      case "Garaža":
+        return AD_CATEGORY.GARAGE.id;
+      case "Vikendica":
+        return AD_CATEGORY.COTTAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "PRODAJA":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "NAJAM":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = SaljicCrawler;

app/helpers/awsEmail.js

@@ -1,202 +0,0 @@
-
-const dotenv = require('dotenv').config();
-const { getRealEstateTypeEnum } = require('./enums');
-const { getRegionName, getMunicipalityName } = require('./codes');
-var AWS = require('aws-sdk');
-const TEMPLATE_NAME = "MarketAlertTemplate"
-
-AWS.config.update({
-  region: process.env.AMAZON_REGION,
-  credentials:
-  {
-    accessKeyId: process.env.AMAZON_ACCES_KEY_ID,
-    secretAccessKey: process.env.AMAZON_SECRET_ACCESS_KEY
-  }
-});
-
-const sendTemplatedEmail = async (email, request) => {
-
-  const params = {
-    Destination: { /* required */
-      CcAddresses: [
-      ],
-      ToAddresses: [
-        email
-      ]
-    },
-    Message: { /* required */
-      Body: { /* required */
-        Html: {
-          Charset: "UTF-8",
-          Data: getGreetingsEmailHTML(request)
-        },
-        Text: {
-          Charset: "UTF-8",
-          Data: getGreetingsEmaiTextVersion(request)
-        }
-      },
-      Subject: {
-        Charset: 'UTF-8',
-        Data: `Javimi Potvrda: ${request.realEstateType} ${getRegionName(request.region)}, ${getMunicipalityName(request.region, request.municipality)}`
-      }
-    },
-    Source: process.env.SOURCE_EMAIL, /* required */
-    ReplyToAddresses: [
-      process.env.SOURCE_EMAIL,
-    ],
-  };
-
-  const sendEmailPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendEmail(params).promise();
-  await sendEmailPromise;
-}
-
-const getGreetingsEmailHTML = (realestateRequest) => {
-  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
-  const gardenSize = realEstateType.hasGardenSize ? `<div><strong>Kvadratura okućnice: Od ${realestateRequest.gardenSizeMin} do ${realestateRequest.gardenSizeMax} m2  </strong></div>` : ``
-
-  return `<h1> Zdravo, 
-  Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima. </h1>
-  <h2> Ovo je tražena nekretnina:  </h2> 
-  <div>
-  <div> <strong>Tip nekretnine: ${realEstateType.title} </strong></div>
-  <div><strong>Područje: ${getRegionName(realestateRequest.region)} </strong></div>
-  <div><strong>Mjesto: ${getMunicipalityName(realestateRequest.region, realestateRequest.municipality)} </strong></div>
-  <div><strong>Kvadratura nekretnine: Od ${realestateRequest.sizeMin} do ${realestateRequest.sizeMax} m2  </strong></div>
-  ${gardenSize}
-  <div><strong>Cijena: ${realestateRequest.priceMin} do ${realestateRequest.priceMax} KM  </strong></div>
-  </div>
-  <div> 
-
-  </div>
-  <div><strong> Ako želis prestati dobijati obavještenja za ovu pretragu klikni ${process.env.APP_URL}/odjava/${realestateRequest.uniqueId} </strong></div>
-  <div><strong>Ako želiš promijeniti uslove pretrage klikni  ${process.env.APP_URL}/pregled/${realestateRequest.uniqueId} </strong></div>
-<h4> Tvoj,
-Javimi tim.  
-</h4>`
-
-}
-
-const getGreetingsEmaiTextVersion = (realestateRequest) => {
-  const realEstateType = getRealEstateTypeEnum(realestateRequest.realEstateType);
-  const gardenSize = realEstateType.hasGardenSize ? "Kvadratura okućnice od " + realestateRequest.gardenSizeMin + " do " + realestateRequest.gardenSizeMax : ""
-
-  const text = "Zdravo, \n Naručio/la si da ti javimo ako se nekretnina pojavi u oglasima \n Ovo je tražena nekretnina: \n , Tip nekretnine: "
-    + realestateRequest.realEstateType + "\n Područje" + getRegionName(realestateRequest.region) + "\n Mjesto " + getMunicipalityName(realestateRequest.region, realestateRequest.municipality)
-    + "\n Kvadratura nekretnine Od " + realestateRequest.sizeMin + " do " + realestateRequest.sizeMaX +
-    + gardenSize
-  "\n Cijena od " + realestateRequest.priceMin + " do " + realestateRequest.priceMax +
-    "\n Ako želis prestati dobijati obavještenja za ovu pretragu klikni" + process.env.APP_URL + "/odjava/" + realestateRequest.uniqueId +
-    "\n Ako želiš promijeniti uslove pretrage klikni " + process.env.APP_URL + "/odpregled/" + realestateRequest.uniqueId +
-    "\n Tvoj,\n Javimi tim"
-
-  return text;
-}
-
-const sendBulkEmail = async (marketAlerts) => {
-
-  try {
-
-    destinations = []
-    groupedEmails = [];
-
-    marketAlerts.forEach(marketAlert => {
-      if (!groupedEmails[marketAlert.email]) {
-        groupedEmails[marketAlert.email] = [];
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
-      } else {
-        groupedEmails[marketAlert.email].push({ url: marketAlert.url, title: marketAlert.title });
-      }
-    });
-
-    for (email in groupedEmails) {
-
-      const url = groupedEmails[email];
-      let repData = `{ "marketAlertUrl":[${toAWSArray(url)}], "favoriteanimal":"yak" }`
-
-      destinations.push({
-        Destination: {
-          ToAddresses: [
-            email
-          ]
-        },
-        ReplacementTemplateData: repData
-      })
-
-    }
-    console.log("AWS EMAIL : Bulk email replacement data:");
-    console.log(destinations);
-
-    var params = {
-      Destinations:
-        destinations,
-      Source: process.env.SOURCE_EMAIL, /* required */
-      Template: TEMPLATE_NAME, /* required */
-      DefaultTemplateData: '{ \"REPLACEMENT_TAG_NAME\":\"REPLACEMENT_VALUE\" }',
-      ReplyToAddresses: [
-        process.env.SOURCE_EMAIL,
-      ]
-    };
-
-    // Create the promise and SES service object
-    const sendPromise = new AWS.SES({ apiVersion: '2010-12-01' }).sendBulkTemplatedEmail(params).promise();
-    const awsResult = await sendPromise;
-    console.log("AWS SES bulk email response");
-    console.log(awsResult);
-
-
-  } catch (e) {
-    console.log("Could not send bulk email", e)
-  }
-
-
-}
-
-const toAWSArray = (urlArray) => {
-  let arrayString = ""
-  urlArray.forEach(element => {
-    const formatetdTitle = element.title.replace(/"/g, "");
-    arrayString = arrayString + `{"url":"${element.url.trim()}" , "title":"${formatetdTitle}"},`
-  });
-
-  return arrayString.slice(0, -1);
-}
-
-const getNotificationEmailHtml = () => {
-  return `<h2> Zdravo, 
-  Pronašli smo nekretninu koju ste tražili. </h2>
-  <h3> Ovo su tražene nekretnine:  </h3> 
-  <div>
-  <div>{{#each marketAlertUrl}}<li><a href="{{url}}">{{title}}</a></li><br />{{/each}}<div/> 
-  <div/>
-  </div>`
-}
-
-const getNotificationEmailText = () => {
-  return ` Zdravo, 
-  Pronašli smo nekretninu koju ste tražili.  Ovo su tražene nekretnine: {{#each marketAlertUrl}} {{url}} {{title}} {{/each}}`
-}
-
-const createMarketAlertEmailTemplate = async () => {
-  const marketAlertTemplate = {
-    Template: {
-      TemplateName: "MarketAlertTemplate",
-      SubjectPart: "Javi mi obavijest",
-      TextPart: getNotificationEmailText(),
-      HtmlPart: getNotificationEmailHtml()
-    }
-  }
-
-  try {
-    const templatePromise = new AWS.SES({ apiVersion: '2010-12-01' }).updateTemplate(marketAlertTemplate).promise();
-    await templatePromise
-
-  } catch (e) {
-    console.log("Could not create MarketAlertEmailTemplate", e);
-  }
-}
-
-module.exports = {
-  sendTemplatedEmail,
-  sendBulkEmail,
-  createMarketAlertEmailTemplate
-};

app/helpers/crawlers/olxClawler.js

@@ -1,235 +0,0 @@
-const fetch = require('node-fetch');
-const cheerio = require('cheerio');
-const { allRERequest, findPointInsideBoundingBox } = require('../db/dbHelper');
-const { getRealEstateTypeEnum } = require('../enums');
-const { getRegion, getMunicipality } = require('../codes')
-const Promise = require("bluebird");
-
-module.exports = class OlxCrawler {
-    //TODO figure best way to handle paging
-    constructor(fromPage = 0, toPage = 10, maxResults = 1000) {
-        this.fromPage = fromPage;
-        this.toPage = toPage;
-        this.maxResults = maxResults;
-    }
-
-    async indexSingle(url, email) {
-        try {
-            const res = await fetch(url);
-            const body = await res.text();
-            const $ = cheerio.load(body);
-
-            //TODO figure out what to do with username
-            const username = $('#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span').text();
-
-            // if (IGNORED_USERNAMES.includes((username || '').toLowerCase())) {
-            //     return null;
-            // }
-
-            //TODO remove properties that are not needed, and add some if they are missing
-            const title = $('#naslovartikla').text().trim();
-            const realEstateType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span').text();
-
-            const price = $('#pc > p:nth-child(2)').text();
-            const size = $('#dodatnapolja1 > div:nth-child(1) > div.df2').text();
-            const rooms = $('#dodatnapolja1 > div:nth-child(2) > div.df2').text();
-            const address = $('#dodatnapolja1 > div:nth-child(5) > div.df2').text();
-            const gardenSize = $('#dodatnapolja1 > div:nth-child(6) > div.df2').text();
-            const location = $('#artikal_glavni_div > div.artikal_lijevo > div.op.pop.mobile-lokacija').attr('data-content');
-
-            const adType = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(2) > div.df2').text();
-            const time = $('time').attr('datetime');
-            const olxId = $('#artikal_glavni_div > div.artikal_lijevo > div:nth-child(15) > div:nth-child(4) > div.df2').text();
-
-            const descriptions = $('.artikal_detaljniopis_tekst');
-            // const floor = $('#dodatnapolja1').find(':contains(Sprat)').last().nextAll().text();
-            const latLngRe = /LatLng\(([0-9]+\.[0-9]+)\,\s+([0-9]+\.[0-9]+)\)/g;
-            const imgRe = /href":("[^"]*")/g;
-            const matches = latLngRe.exec(body);
-            let lng = '',
-                lat = '';
-
-
-            const parseRooms = (rooms) => parseInt([...rooms].filter(c => !isNaN(c)).filter(c => c.trim()).join())
-            const parsePrice = (price) => parseFloat(price.replace(".", ""))
-
-
-            // TODO we dont save images ??
-
-            // const images = [];
-            // const imgMatches = body.match(imgRe);
-
-            // for (let i = 0; imgMatches && i < imgMatches.length; i++) {
-            //     let img = imgMatches[i].replace("href\":", "")
-            //     img = img.replace("\"", "");
-            //     img = img.replace("\"", "");
-            //     images.push(img);
-            // }
-
-            // const uploadPromises = images.map(img => {
-            //     const imgFixed = eval(`'${img}'`);
-            //     return cloudinary.uploader.upload(eval(`'${img}'`));
-            // });
-
-            // const uploadResults = await Promise.all(uploadPromises);
-            // const cloudinaryImages = uploadResults.map(ur => ur.url);
-
-            if (matches && matches.length >= 3) {
-                lat = matches[1];
-                lng = matches[2];
-            }
-
-            const parsedPrice = parsePrice(price);
-
-            const locationArray = location.split(",");
-            const region = locationArray[0];
-            const municipality = locationArray[1];
-
-            const data = {
-                realEstateType: this.getCategoryId(realEstateType),
-                email: email,
-                olxId: olxId,
-                // category: category,
-                url,
-                title,
-                price: isNaN(parsedPrice) ? 0 : parsedPrice,
-                size: parseFloat(size),
-                gardenSize: isNaN(parseFloat(gardenSize)) ? 0 : parseFloat(gardenSize),
-                address,
-                region,
-                municipality,
-                // adType: AD_TYPE_SALE,
-                time,
-                shortDescription: descriptions.first().text(),
-                longDescription: descriptions.last().text(),
-                lat,
-                lng,
-                loc: [parseFloat(lat), parseFloat(lng)],
-                // images: cloudinaryImages
-            };
-
-            return data;
-        } catch (e) {
-            console.error('Exception caught: ' + e.message);
-        }
-
-        return null;
-    }
-
-    async indexPage(olxUrl, maxResults = 1000) {
-        try {
-            //TODO fix paging
-            // console.log('Starting to index page: ' + pageNr);
-            // const url = `http://www.olx.ba/pretraga?vrsta=samoprodaja&sort_order=desc&kategorija=23&sort_po=datum&kanton=9&stranica=${pageNr}`;
-
-            const res = await fetch(olxUrl.url);
-            const body = await res.text();
-            const $ = cheerio.load(body);
-            const hrefs = [];
-            const results = [];
-
-            $('#rezultatipretrage').find('.listitem').each((i, elem) => {
-                const href = $(elem).find('a').first().attr('href');
-                hrefs.push(href);
-            });
-
-            let actualNoOfResults = (hrefs.length <= maxResults) ? hrefs.length : maxResults;
-
-            for (let i = 0; i < hrefs.length; i++) {
-                console.log(`indexing: ${hrefs[i]}`);
-
-                const singleData = await this.indexSingle(hrefs[i], olxUrl.email);
-
-                if (singleData) {
-                    results.push(singleData);
-                }
-            }
-
-            return results;
-        } catch (e) {
-            console.error('Exception caught:' + e);
-        }
-    }
-
-    getCategoryId(category) {
-
-        switch (category) {
-            case 'Stanovi':
-                return 'stan';
-
-            case 'Vikendice':
-                return 'vikendica'
-
-            case 'Kuće':
-                return 'kuca';
-
-            default:
-                return '';
-        }
-    }
-
-    async indexPages(urls, start, end, maxResults = 1000) {
-        //TODO fix paging
-        // let results = {};
-        // for (let i = start; i <= end; i++) {
-        //     let result = await this.indexPage(i, maxResults);
-        //     Object.assign(results, result)
-        // }
-        // return results;
-
-        let results = [];
-        for (let url of urls) {
-            let result = await this.indexPage(url, maxResults);
-            results.push(result);
-        }
-        return results;
-    }
-
-    async crawl() {
-        console.log("OLX CRAWLER: start crawl");
-
-        const filteredResults = [];
-        const realestateRequests = await allRERequest();
-        console.log("OLX CRAWLER: found " + realestateRequests.length + "subscribed RealEstateRequests");
-        const urls = this.createRequestUrls(realestateRequests);
-        let results = await this.indexPages(urls, this.fromPage, this.toPage, this.maxResults);
-
-        for (const result of results) {
-            for (const finalResult of result) {
-                if (finalResult.lat !== undefined && finalResult.lat !== null && finalResult.lat !== "") {
-                    const pointInsideBoundingBox = await findPointInsideBoundingBox([finalResult.lng, finalResult.lat], finalResult.email);
-
-                    if (pointInsideBoundingBox[0].length !== 0) {
-                        filteredResults.push(finalResult);
-                    }
-                }
-            }
-        }
-        console.log("OLX CRAWLER: number of olx crawler results, after geo location filtering: " + filteredResults.length);
-        return filteredResults;
-    }
-
-    createRequestUrls(realestateRequests) {
-        const urls = []
-
-        for (const request of realestateRequests) {
-            const realsestateType = "kategorija=" + getRealEstateTypeEnum(request.realEstateType).olxCategory;
-            const region = "kanton=" + getRegion(request.region).olxid;
-            const municipality = "grad%5B%5D=" + getMunicipality(request.region, request.municipality).olxid;
-            const sizeMin = "kvadrata_min=" + request.sizeMin;
-            const sizeMax = "kvadrata_max=" + request.sizeMax;
-            const priceMin = "od=" + request.priceMin;
-            const priceMax = "do=" + request.priceMax;
-
-            const olxUrl = {
-                url: `https://www.olx.ba/pretraga?${realsestateType}&id=2&stanje=0&vrstapregleda=tabela&sort_order=desc&${region}&${municipality}&${priceMin}&${priceMax}&vrsta=samoprodaja&${sizeMin}&${sizeMax}`,
-                email: request.email
-            }
-            console.log(olxUrl.url);
-            urls.push(olxUrl);
-        }
-
-        return urls;
-    }
-};
-

app/helpers/db/dbHelper.js

@@ -1,54 +0,0 @@
-const db = require('../../models/index');
-
-/**
- * Find all subscribed RealEstateRequests 
- */
-const allRERequest = async () => {
-    return await db.RealEstateRequest.findAll({
-        where: {
-            subscribed: true
-        }
-      });
-}
-
-/**
- * Find all , or all depending on notified bolean marketalerts, and order them by email
- * 
- * @param fechAll bolean
- * @param notified bolean
- * 
- * @returns array of MarketAlerts
- */
-const allMarketAlerts = async (fetchAll, notified) => {
-
-    let queryObject = {
-        order: [
-            ['email', 'DESC'],
-        ]
-    }
-
-    if (!fetchAll){
-        queryObject.where = {
-            notified: notified
-        }
-    }
-
-    return await db.MarketAlert.findAll(queryObject);
-  }
-  
-/**
- * Find all unnotified marketalerts 
- *  @param latLng array
- *  @param email string
- * 
- * @returns array of MarketAlerts
- */
-const findPointInsideBoundingBox = async (latLng, email) => {
-  return await db.sequelize.query(`SELECT * FROM "RealEstateRequests" WHERE email = '${email}' AND subscribed = true AND ST_Contains("RealEstateRequests".bounding_box, ST_GEOMFROMTEXT('POINT (${latLng[0]} ${latLng[1]})'))`);
-}
-
-module.exports = {
-  allRERequest,
-  allMarketAlerts,
-  findPointInsideBoundingBox
-};

app/helpers/db/priceHistory.js

@@ -0,0 +1,20 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const bulkUpsertPriceHistory = async priceHistoryData => {
+  try {
+    const order = [["realEstateId", "desc"]];
+
+    return await db.PriceHistory.bulkCreate(priceHistoryData, {
+      order,
+      ignoreDuplicates: true
+    });
+  } catch (e) {
+    console.log("Error bulk upserting priceHistory : ", e);
+  }
+};
+
+module.exports = {
+  bulkUpsertPriceHistory
+};

app/helpers/db/realEstate.js

@@ -0,0 +1,352 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+const { AD_CATEGORY } = require("../../common/enums");
+
+const bulkUpsertRealEstates = async realEstateData => {
+  try {
+    const fieldsToUpdateIfDuplicate = [
+      "realEstateType",
+      "adType",
+      "price",
+      "area",
+      "streetNumber",
+      "streetName",
+      "locality",
+      "municipality",
+      "city",
+      "region",
+      "entity",
+      "country",
+      "locationLat",
+      "locationLong",
+      "title",
+      "shortDescription",
+      "longDescription",
+      "gardenSize",
+      "adStatus",
+      "updatedAt",
+      "renewedDate",
+      "numberOfRooms",
+      "numberOfFloors",
+      "floor",
+      "accessRoadType",
+      "heatingType",
+      "furnishingType",
+      "balcony",
+      "newBuilding",
+      "elevator",
+      "water",
+      "electricity",
+      "drainageSystem",
+      "registeredInZkBooks",
+      "recentlyAdapted",
+      "parking",
+      "garage",
+      "gas",
+      "antiTheftDoor",
+      "airCondition",
+      "phoneConnection",
+      "cableTV",
+      "internet",
+      "basementAttic",
+      "storeRoom",
+      "videoSurveillance",
+      "alarm",
+      "suitableForStudents",
+      "includingBills",
+      "animalsAllowed",
+      "pool",
+      "urbanPlanPermit",
+      "buildingPermit",
+      "utilityConnection",
+      "distanceToRiver",
+      "numberOfViewsAgency"
+    ];
+    const order = [["updatedAt", "desc"]];
+
+    return await db.RealEstate.bulkCreate(realEstateData, {
+      updateOnDuplicate: fieldsToUpdateIfDuplicate,
+      returning: true,
+      order
+    });
+  } catch (e) {
+    console.log("Error bulk upserting realEstates : ", e);
+  }
+};
+
+const getRealEstateById = async id => {
+  return db.RealEstate.findByPk(id);
+};
+
+const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
+  const {
+    priceMin,
+    priceMax,
+    sizeMin,
+    sizeMax,
+    adType,
+    realEstateType,
+    areaToSearch,
+    gardenSizeMin,
+    gardenSizeMax,
+    numberOfRoomsMin,
+    numberOfRoomsMax,
+    numberOfFloorsMin,
+    numberOfFloorsMax,
+    floorMin,
+    floorMax,
+    includeIncompleteAds,
+    includeWithoutPrice,
+    balcony,
+    elevator,
+    newBuilding,
+    accessRoadType
+  } = searchRequest;
+
+  //Needed for defining which attribute should exist or not
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+
+  const longitudeColumn = sequelize.col("locationLong");
+  const latitudeColumn = sequelize.col("locationLat");
+
+  const pointGeometry = sequelize.fn(
+    "ST_Point",
+    longitudeColumn,
+    latitudeColumn
+  );
+  const pointWithSRID = sequelize.fn("ST_SetSRID", pointGeometry, 4326);
+  const areaToSearchAsGeometry = sequelize.fn(
+    "ST_GeomFromGeoJSON",
+    JSON.stringify(areaToSearch)
+  );
+  const areaToSearchWithSRID = sequelize.fn(
+    "ST_SetSRID",
+    areaToSearchAsGeometry,
+    4326
+  );
+  const contains = sequelize.fn(
+    "ST_Contains",
+    areaToSearchWithSRID,
+    pointWithSRID
+  );
+
+  const geoSearchQueryPart = sequelize.where(contains, true);
+
+  //General queries contain only attributes that are defined for every searchreq
+
+  //Query for case of complete ads
+  const query = {
+    adType,
+    realEstateType,
+    area: {
+      [Op.lte]: sizeMax,
+      [Op.gte]: sizeMin
+    },
+    [Op.and]: geoSearchQueryPart
+  };
+
+  //Query for case of incomplete ads
+  const queryIncludeIncomplete = {
+    adType,
+    realEstateType,
+    area: {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: sizeMax,
+          [Op.gte]: sizeMin
+        },
+        [Op.is]: null
+      }
+    },
+    [Op.and]: geoSearchQueryPart
+  };
+
+  //Is user unchecked includeWithoutPrice FALSE then it shouldn't return null values of price
+  //If not then null values are accepted (this is DEFAULT)
+  //includeIncpompleteAds does not have effect on price query
+  if (includeWithoutPrice) {
+    query.price = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    };
+    queryIncludeIncomplete.price = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    };
+  } else {
+    query.price = {
+      [Op.and]: {
+        [Op.lte]: priceMax,
+        [Op.gte]: priceMin
+      }
+    };
+    queryIncludeIncomplete.price = {
+      [Op.and]: {
+        [Op.lte]: priceMax,
+        [Op.gte]: priceMin
+      }
+    };
+  }
+  //Every other attribute is checked separately and included in query only if it is defined for real estate type
+
+  if (
+    realEstateTypeObject.hasGardenSize &&
+    gardenSizeMax != null &&
+    gardenSizeMin != null
+  ) {
+    query.gardenSize = {
+      [Op.lte]: gardenSizeMax,
+      [Op.gte]: gardenSizeMin
+    };
+    queryIncludeIncomplete.gardenSize = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: gardenSizeMax,
+          [Op.gte]: gardenSizeMin
+        },
+        [Op.is]: null
+      }
+    };
+  }
+
+  if (
+    realEstateTypeObject.hasNumberOfRoom &&
+    numberOfRoomsMin != null &&
+    numberOfRoomsMax != null
+  ) {
+    query.numberOfRooms = {
+      [Op.lte]: numberOfRoomsMax,
+      [Op.gte]: numberOfRoomsMin
+    };
+    queryIncludeIncomplete.numberOfRooms = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: numberOfRoomsMax,
+          [Op.gte]: numberOfRoomsMin
+        },
+        [Op.is]: null
+      }
+    };
+  }
+
+  if (
+    realEstateTypeObject.hasNumberOfFloors &&
+    numberOfFloorsMin != null &&
+    numberOfFloorsMax != null
+  ) {
+    query.numberOfFloors = {
+      [Op.lte]: numberOfFloorsMax,
+      [Op.gte]: numberOfFloorsMin
+    };
+    queryIncludeIncomplete.numberOfFloors = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: numberOfFloorsMax,
+          [Op.gte]: numberOfFloorsMin
+        },
+        [Op.is]: null
+      }
+    };
+  }
+
+  if (
+    realEstateTypeObject.hasFloorProp &&
+    floorMin != null &&
+    floorMax != null
+  ) {
+    query.floor = {
+      [Op.lte]: floorMax,
+      [Op.gte]: floorMin
+    };
+    queryIncludeIncomplete.floor = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: floorMax,
+          [Op.gte]: floorMin
+        },
+        [Op.is]: null
+      }
+    };
+  }
+  //Logic for balcony, newBuilding and elevator from users side
+  //If true is checked, then I want characteristic to be true but,
+  //if it is not checked, then I dont care - it can be null or false or true
+  if (realEstateTypeObject.hasBalconyProp && balcony === true) {
+    query.balcony = {
+      [Op.eq]: balcony
+    };
+    queryIncludeIncomplete.balcony = {
+      [Op.or]: {
+        [Op.eq]: balcony,
+        [Op.is]: null
+      }
+    };
+  }
+
+  if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) {
+    query.newBuilding = {
+      [Op.eq]: newBuilding
+    };
+    queryIncludeIncomplete.newBuilding = {
+      [Op.or]: {
+        [Op.eq]: newBuilding,
+        [Op.is]: null
+      }
+    };
+  }
+
+  if (realEstateTypeObject.hasElevatorProp && elevator === true) {
+    query.elevator = {
+      [Op.eq]: elevator
+    };
+    queryIncludeIncomplete.elevator = {
+      [Op.or]: {
+        [Op.eq]: elevator,
+        [Op.is]: null
+      }
+    };
+  }
+  //If user wants 'ANY' road type acces then it is not included in query -
+  //returns every road type and null values
+  if (accessRoadType !== "ANY") {
+    query.accessRoadType = {
+      [Op.eq]: accessRoadType
+    };
+    queryIncludeIncomplete.accessRoadType = {
+      [Op.or]: {
+        [Op.eq]: accessRoadType,
+        [Op.is]: null
+      }
+    };
+  }
+
+  //When includeIncompleteAds are not defined - null it will consider it true
+  const order = [["updatedAt", "desc"]];
+
+  return db.RealEstate.findAll({
+    where:
+      includeIncompleteAds || includeIncompleteAds == null
+        ? queryIncludeIncomplete
+        : query,
+    limit: maxResults,
+    order
+  });
+};
+
+module.exports = {
+  bulkUpsertRealEstates,
+  getRealEstateById,
+  findRealEstatesForSearchRequest
+};

app/helpers/db/searchRequest.js

@@ -0,0 +1,337 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+const { AD_CATEGORY } = require("../../common/enums");
+
+const getSearchRequest = async searchRequestId => {
+  try {
+    return await db.SearchRequest.findByPk(searchRequestId);
+  } catch (error) {
+    console.log("searchrequest.js", error);
+    return null;
+  }
+};
+
+const createSearchRequest = async (searchRequestFields = {}) => {
+  return await db.SearchRequest.create(searchRequestFields);
+};
+
+const findSearchRequestsForRealEstate = async realEstate => {
+  const {
+    price,
+    area,
+    adType,
+    realEstateType,
+    locationLat,
+    locationLong,
+    accessRoadType,
+    balcony,
+    newBuilding,
+    elevator,
+    gardenSize,
+    numberOfRooms,
+    numberOfFloors,
+    floor
+  } = realEstate;
+
+  if (!locationLat || !locationLong) {
+    return [];
+  }
+
+  const stGeometry = sequelize.fn(
+    "ST_GEOMFROMTEXT",
+    `POINT (${locationLong} ${locationLat})`,
+    4326
+  );
+  const areaToSearchColumn = sequelize.col("areaToSearch");
+  const contains = sequelize.fn("ST_Contains", areaToSearchColumn, stGeometry);
+
+  const geoSearchQueryPart = sequelize.where(contains, true);
+
+  //Needed for defining which attribute should exist or not
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+
+  // ?? Needed to decide on including incomplete RealEstates data
+  let checkForIncompleteWanted = false;
+
+  //Attributes are checked separately to make different query parts
+
+  //If real estate price is number then it searches for req that have priceMin and priceMax
+  //If real estate price is null it searches for req that accept ads without price
+  //User always defines price and area (sliders) - not null in search req
+  let priceQuery = {};
+  if (price != null) {
+    priceQuery = {
+      [Op.and]: [
+        {
+          priceMin: {
+            [Op.lte]: price
+          }
+        },
+        {
+          priceMax: {
+            [Op.gte]: price
+          }
+        }
+      ]
+    };
+  } else {
+    priceQuery = {
+      includeWithoutPrice: {
+        [Op.eq]: true
+      }
+    };
+  }
+
+  let areaQuery = {};
+  if (area != null) {
+    areaQuery = {
+      [Op.and]: [
+        {
+          sizeMin: {
+            [Op.lte]: area
+          }
+        },
+        {
+          sizeMax: {
+            [Op.gte]: area
+          }
+        }
+      ]
+    };
+  } else {
+    checkForIncompleteWanted = true;
+  }
+
+  //Other attributes can be defined or not depending on RealEstate type
+  //we check what to include in query based on real estate type object
+  let gardenSizeQuery = {};
+  if (realEstateTypeObject.hasGardenSize) {
+    if (gardenSize != null) {
+      gardenSizeQuery = {
+        [Op.and]: [
+          {
+            gardenSizeMin: {
+              [Op.lte]: gardenSize
+            }
+          },
+          {
+            gardenSizeMax: {
+              [Op.gte]: gardenSize
+            }
+          }
+        ]
+      };
+    } else {
+      checkForIncompleteWanted = true;
+    }
+  }
+
+  let numberOfRoomsQuery = {};
+  if (realEstateTypeObject.hasNumberOfRoom) {
+    if (numberOfRooms != null) {
+      //If real estate has defined number of rooms ex. 3 it returns req
+      // that accepts 3 rooms or ones that don't have defined number - null
+      //Ex. they didnt choose advanced filters at all
+      numberOfRoomsQuery = {
+        [Op.and]: [
+          {
+            numberOfRoomsMin: {
+              [Op.or]: {
+                [Op.lte]: numberOfRooms,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            numberOfRoomsMax: {
+              [Op.or]: {
+                [Op.gte]: numberOfRooms,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      // If real estate dont have defined number of rooms ex. null
+      //It returns all search requests except for ones that dont want incpomlete ads
+      numberOfRoomsQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+  //Same logic for number of Floors and floors
+  let numberOfFloorsQuery = {};
+  if (realEstateTypeObject.hasNumberOfFloors) {
+    if (numberOfFloors != null) {
+      numberOfFloorsQuery = {
+        [Op.and]: [
+          {
+            numberOfFloorsMin: {
+              [Op.or]: {
+                [Op.lte]: numberOfFloors,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            numberOfFloorsMax: {
+              [Op.or]: {
+                [Op.gte]: numberOfFloors,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      numberOfFloorsQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+  let floorQuery = {};
+  if (realEstateTypeObject.hasFloorProp) {
+    if (floor != null) {
+      floorQuery = {
+        [Op.and]: [
+          {
+            floorMin: {
+              [Op.or]: {
+                [Op.lte]: floor,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            floorMax: {
+              [Op.or]: {
+                [Op.gte]: floor,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      floorQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+
+  //Logic for balcony, newBuilding and elevator
+  //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
+  //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
+  //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
+  //If real estate characteristic = null, dont know if true or false, find all req except ones that dont want incomplete ads
+  let balconyQuery = {};
+  if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
+    if (balcony === false) {
+      balconyQuery = {
+        balcony: {
+          [Op.ne]: true
+        }
+      };
+    } else if (balcony === null) {
+      balconyQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+  let newBuildingQuery = {};
+  if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) {
+    if (newBuilding === false) {
+      newBuildingQuery = {
+        newBuilding: {
+          [Op.ne]: true
+        }
+      };
+    } else if (newBuilding === null) {
+      newBuildingQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+  let elevatorQuery = {};
+  if (realEstateTypeObject.hasElevatorProp && elevator !== true) {
+    if (elevator === false) {
+      elevatorQuery = {
+        elevator: {
+          [Op.ne]: true
+        }
+      };
+    } else if (elevator === null) {
+      elevatorQuery = {
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
+      };
+    }
+  }
+
+  //General query consists of each individual query
+  const query = {
+    adType,
+    realEstateType,
+    subscribed: true,
+    [Op.and]: [
+      geoSearchQueryPart,
+      priceQuery,
+      areaQuery,
+      gardenSizeQuery,
+      numberOfRoomsQuery,
+      numberOfFloorsQuery,
+      floorQuery,
+      balconyQuery,
+      newBuildingQuery,
+      elevatorQuery
+    ]
+  };
+
+  //AccessRoadType is defined - should exists for each ad and estate type
+  if (accessRoadType != null) {
+    query.accessRoadType = {
+      [Op.or]: {
+        [Op.like]: "ANY",
+        [Op.eq]: accessRoadType
+      }
+    };
+  } else {
+    //Null values are returned for user request that wanted ANY acces road type
+    query.accessRoadType = {
+      [Op.eq]: "ANY"
+    };
+  }
+  //Tag to check if incomplete ads are accepted in query which is default
+  if (checkForIncompleteWanted) {
+    query.includeIncompleteAds = {
+      [Op.or]: {
+        [Op.eq]: true,
+        [Op.is]: null
+      }
+    };
+  }
+
+  return await db.SearchRequest.findAll({
+    where: query
+  });
+};
+
+module.exports = {
+  getSearchRequest,
+  createSearchRequest,
+  findSearchRequestsForRealEstate
+};

app/helpers/db/searchRequestMatch.js

@@ -0,0 +1,94 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
+
+const findRealEstatesForSearchRequest = async searchRequestId => {
+  const query = {
+    searchRequestId
+  };
+
+  const realEstatesModel = { model: db.RealEstate, as: "realEstates" };
+  const order = [[realEstatesModel, "updatedAt", "desc"]];
+  const include = [realEstatesModel];
+
+  const matches = await db.SearchRequestMatch.findAll({
+    where: query,
+    include,
+    order
+  });
+
+  const matchingRealEstates = [];
+  for (const match of matches) {
+    matchingRealEstates.push(...match.realEstates);
+  }
+
+  return matchingRealEstates;
+};
+
+const findNotNotifiedMatches = async () => {
+  const query = {
+    notified: false
+  };
+
+  const searchRequestsModel = { model: db.SearchRequest, as: "searchRequests" };
+  const realEstateModel = { model: db.RealEstate, as: "realEstates" };
+  const include = [searchRequestsModel, realEstateModel];
+
+  const matchingRecords = await db.SearchRequestMatch.findAll({
+    where: query,
+    include
+  });
+
+  return matchingRecords;
+};
+const findAllRequestsForCheckUp = async () => {
+  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
+  //The ones that received notification for real estate CHECK_UP_DAYS days from now
+  const date = new Date();
+  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
+  date.setDate(checkUpDate);
+  const dateQuery = {
+    createdAt: {
+      [Op.gte]: date
+    }
+  };
+
+  const excludedMatches = await db.SearchRequestMatch.findAll({
+    attributes: ["searchRequestId"],
+    where: dateQuery,
+    order: [["searchRequestId", "ASC"]]
+  });
+
+  const excludedRequestsAll = excludedMatches.map(match => {
+    return match.dataValues.searchRequestId;
+  });
+  //Removing duplicate search request id-s for optimization
+  const excludedRequests = [...new Set(excludedRequestsAll)];
+
+  const query = {
+    subscribed: true,
+    id: {
+      [Op.notIn]: excludedRequests
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: query
+  });
+
+  return allRequestsForCheckUp;
+};
+
+const addMatches = async matchingRecords => {
+  return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
+    ignoreDuplicates: true
+  });
+};
+
+module.exports = {
+  findRealEstatesForSearchRequest,
+  addMatches,
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp
+};

app/helpers/email.js

@@ -1,7 +1,6 @@
-
-const isValidEmail = (email) => {
-  const simpleEmailRegex = /^.+@.+\..+$/;
-  return (email && email.length < 250 && simpleEmailRegex.test(email));
+const isValidEmail = email => {
+  const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+  return email && email.length < 250 && simpleEmailRegex.test(email);
 };
 
 module.exports = {

app/helpers/emailContentGenerator.js

@@ -0,0 +1,210 @@
+"use strict";
+
+const {
+  MAX_REAL_ESTATES_IN_EMAIL,
+  APP_URL,
+  STAGING
+} = require("../config/appConfig");
+const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
+
+//Tag to recognize staging from development
+const stagingTag = STAGING ? "[STAGING] " : "";
+
+const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
+  return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>
+    <div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
+     <div>Ako želite pogledati ili promijeniti uslove za ovu pretragu, <a href="${APP_URL}/pregled/${searchRequestId}">pogledajte ovdje</a></div>
+     <br/> 
+     <strong>Vaš,<br/>Kivi tim</strong>`;
+};
+
+const generateRealEstateLinks = realEstates => {
+  let realEstateLinks = "";
+  for (const realEstate of realEstates) {
+    const { id: realEstateId, title } = realEstate;
+
+    realEstateLinks += `<li><a href="${APP_URL}/redirect/${realEstateId}">${title}</a></li>`;
+  }
+  return realEstateLinks;
+};
+
+const generateNotificationEmail = (
+  realEstates,
+  searchRequestId,
+  noAllRealEstates,
+  dailyNotification = false
+) => {
+  const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
+
+  const realEstatesToShow = truncateList
+    ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
+    : realEstates;
+
+  const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
+
+  const emailFrequencyTitle = dailyNotification
+    ? EMAIL_FREQUENCY.DAILY.title
+    : EMAIL_FREQUENCY.ASAP.title;
+
+  const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
+  const moreRealEstates = `<div>Kompletan spisak nekretnina (${noAllRealEstates}) možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
+  const emailFooter = generateEmailFooter(searchRequestId, emailFrequencyTitle);
+  const asapMessageBody =
+    realEstates.length > 1
+      ? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
+      : "Pronašli smo nekretninu koja odgovara Vašoj pretrazi";
+
+  const dailyMessageBody =
+    realEstates.length > 1
+      ? "U posljednja 24h objavljene su sljedeće nekretnine koje odgovaraju uslovima Vaše pretrage"
+      : "U posljednja 24h objavljena je sljedeća nekretnina koja odgovara uslovima Vaše pretrage";
+
+  const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
+
+  return `<h3>${stagingTag}Zdravo</h3>
+  <h4>${messageBody}</h4>
+  <div>
+  ${realEstateLinks}
+  <div/>
+  ${moreRealEstates}
+  </div>
+  <br/>
+  ${emailFooter}`;
+};
+
+const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  let adTypeTitle = "";
+  switch (searchRequest.adType) {
+    case AD_TYPE.AD_TYPE_SALE.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
+      break;
+    case AD_TYPE.AD_TYPE_RENT.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
+      break;
+    default:
+      adTypeTitle = "-";
+      break;
+  }
+  let emailFrequencyTitle;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.ASAP.title;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.DAILY.title;
+      break;
+  }
+
+  const {
+    id,
+    gardenSizeMin,
+    gardenSizeMax,
+    sizeMin,
+    sizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest;
+
+  const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
+
+  const instantRealEstatesText = `<br/>
+       <div>
+        U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
+        ${realEstateLinks}
+       </div>`;
+
+  const gardenSize = realEstateType.hasGardenSize
+    ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
+    : ``;
+
+  const emailFooter = generateEmailFooter(id, emailFrequencyTitle);
+
+  return `<h3>${stagingTag}Zdravo</h3>
+  <div>Naručili ste da Vam javimo ako se nekretnina sa navedenim uslovima pojavi u oglasima:</div>
+  <br/>
+  <div>
+    <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Vrsta oglasa: </strong>${adTypeTitle}</div>
+    <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
+    ${gardenSize}
+    <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
+  </div>
+  ${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
+  <br/>
+  ${emailFooter}`;
+};
+
+const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
+  if (numberOfRealEstates === 1) {
+    return `${stagingTag}Kivi: ${singleRealEstateTitle}`;
+  }
+
+  const leastSignificantDigit = numberOfRealEstates % 10;
+  const numberWithoutLastDigit = Math.floor(numberOfRealEstates / 10);
+  const secondLeastSignificantDigit = numberWithoutLastDigit % 10;
+
+  if (leastSignificantDigit === 1 && secondLeastSignificantDigit !== 1) {
+    return `${stagingTag}Kivi : ${numberOfRealEstates} nova nekretnina`;
+  }
+
+  if (
+    leastSignificantDigit >= 2 &&
+    leastSignificantDigit <= 4 &&
+    secondLeastSignificantDigit !== 1
+  ) {
+    return `${stagingTag}Kivi: ${numberOfRealEstates} nove nekretnine`;
+  }
+
+  return `${stagingTag}Kivi: ${numberOfRealEstates} novih nekretnina`;
+};
+
+const generateCheckUpEmail = searchRequest => {
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  const {
+    id,
+    gardenSizeMin,
+    gardenSizeMax,
+    sizeMin,
+    sizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest;
+
+  let emailFrequencyTitle;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.ASAP.title;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.DAILY.title;
+      break;
+  }
+
+  const gardenSize = realEstateType.hasGardenSize
+    ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
+    : ``;
+
+  const emailFooter = generateEmailFooter(id, emailFrequencyTitle);
+
+  return `<h3>${stagingTag}Zdravo</h3>
+  <div><strong>Kivi tim traži nekretnine za Vas i kada to ne vidite.</strong></div>
+  <br />
+  <div>Vaša trenutno aktivna pretraga je:</div>
+  <br/>
+  <div>
+    <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
+    ${gardenSize}
+    <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
+  </div>
+  <br/>
+  ${emailFooter}`;
+};
+
+module.exports = {
+  generateNotificationEmail,
+  generateNewSearchRequestEmail,
+  generateEmailSubject,
+  generateCheckUpEmail
+};

app/helpers/enums.js

@@ -1,57 +0,0 @@
-const realEstateTypes = [
-    { title: "Kuća", id: "kuca", hasGardenSize: true, olxCategory: 24 },
-    { title: "Stan", id: "stan", hasGardenSize: false, olxCategory: 23},
-    { title: "Vikendica", id: "vikendica", hasGardenSize: true, olxCategory: 26 }
-];
-
-const sizes = [
-    { title: "do 50 m2", id: "50m2" },
-    { title: "do 75 m2", id: "75m2" },
-    { title: "do 100 m2", id: "100m2" },
-    { title: "do 150 m2", id: "150m2" },
-    { title: "do 200 m2", id: "200m2" },
-    { title: "preko 200 m2", id: "moreThan200m2" }
-];
-
-const gardenSizes = [
-    { title: "do 100 m2", id: "100m2" },
-    { title: "do 500 m2", id: "500m2" },
-    { title: "do 1 dunum", id: "1000m2" },
-    { title: "do 2 dunuma", id: "2000m2" },
-    { title: "do 3 dunuma", id: "3000m2" },
-    { title: "preko 3 dunuma", id: "moreThan3000m2" }
-];
-
-const prices = [
-    { title: "do 50 000 KM", id: "50kKM" },
-    { title: "do 100 000 KM", id: "100kKM" },
-    { title: "do 150 000 KM", id: "150kKM" },
-    { title: "do 200 000 KM", id: "200kKM" },
-    { title: "do 250 000 KM", id: "250kKM" },
-    { title: "preko 250 000 KM", id: "moreThan250kKM" }
-];
-
-const getEnumObject = (enumType, enumId) => {
-    return enumType.find(enumValue => enumValue.id === enumId);
-};
-
-const getRealEstateTypeEnum = (enumId) => {
-    return getEnumObject(realEstateTypes, enumId) || null;
-}
-
-const getEnumTypeTitle = (enumType, enumId) => {
-    const enumObject = getEnumObject(enumType, enumId);
-    if (!enumObject){
-        return null;
-    }
-    return enumObject.title;
-};
-
-module.exports = {
-    realEstateTypes,
-    sizes,
-    gardenSizes,
-    prices,
-    getRealEstateTypeEnum,
-    getEnumTypeTitle,
-};

app/helpers/fetchWrapper.js

@@ -0,0 +1,13 @@
+const nodeFetch = require("node-fetch");
+const { USER_AGENT } = require("../config/appConfig");
+
+const fetch = async (url, options = {}) => {
+  const newOptions = Object.assign({}, options);
+  if (!newOptions["headers"]) {
+    newOptions["headers"] = {};
+  }
+  newOptions["headers"]["User-Agent"] = USER_AGENT;
+  return nodeFetch(url, newOptions);
+};
+
+module.exports = fetch;

app/helpers/forceSSL.js

@@ -0,0 +1,25 @@
+/**
+ * Force load with https on production environment
+ * https://devcenter.heroku.com/articles/http-routing#heroku-headers
+ */
+module.exports = function(environments, status) {
+  environments = environments || ["production"];
+  status = status || 301;
+  // console.log("New force SSL ");
+  // console.log("\tenvs : ", environments);
+  // console.log("\tstatus: ", status);
+  // console.log("\tENV : ", process.env.NODE_ENV);
+  return function(req, res, next) {
+    if (environments.indexOf(process.env.NODE_ENV) >= 0) {
+      if (req.headers["x-forwarded-proto"] !== "https") {
+        const urlToRedirectTo = `https://${req.hostname}${req.originalUrl}`;
+        // console.log("\tRedirect :", urlToRedirectTo);
+        res.redirect(status, urlToRedirectTo);
+      } else {
+        next();
+      }
+    } else {
+      next();
+    }
+  };
+};

app/helpers/url.js

@@ -1,12 +1,12 @@
-const db = require('../models/index');
+const { getSearchRequest } = require("./db/searchRequest");
 
-const currentRERequest = async (req) => {
-  const uniqueId = req.params['request_id'];
-  if(!uniqueId) return null;
+const currentSearchRequest = async req => {
+  const searchRequestId =
+    req && req.params ? req.params["searchRequestId"] : null;
+  if (!searchRequestId) return null;
 
-  const request = await db.RealEstateRequest.findOne({ where: {uniqueId} });
-  return request;
+  return await getSearchRequest(searchRequestId);
 };
 module.exports = {
-  currentRERequest,
+  currentSearchRequest
 };

app/lib/areThereAnyNewItems.js

@@ -1,10 +0,0 @@
-const convertToDate = require("./convertToDate");
-
-function areThereAnyNewItems(lastItemDate, controlDate) {
-  if (!lastItemDate) {
-    return true;
-  }
-  return new Date(controlDate) < convertToDate(lastItemDate);
-}
-
-module.exports = areThereAnyNewItems;

app/lib/convertToDate.js

@@ -1,13 +0,0 @@
-function convertToDate(date) {
-  const [dan, mjesec, godina] = date
-    .split(". u ")[0]
-    .split(".")
-    .map(el => Number(el));
-  const [sati, minute] = date
-    .split(". u ")[1]
-    .split(":")
-    .map(el => Number(el));
-  return new Date(godina, mjesec, dan, sati, minute);
-}
-
-module.exports = convertToDate;

app/lib/scrapTheItems.js

@@ -1,42 +0,0 @@
-let fetch = require("node-fetch");
-let cheerio = require("cheerio");
-const areThereAnyNewItems = require("./areThereAnyNewItems");
-
-async function scrapTheItems(url, controlDate, noNewItems = false) {
-  let items = [];
-  let response = await fetch(url);
-  const body = await response.text();
-  const $ = cheerio.load(body);
-  $("#rezultatipretrage")
-    .find(".listitem")
-    .each(async (index, elem) => {
-      if (noNewItems) return;
-      const itemDate = $(elem)
-        .find(".cijena > .datum > div")
-        .first()
-        .attr("data-cijelidatum");
-
-      if (controlDate && !areThereAnyNewItems(itemDate, controlDate)) {
-        noNewItems = true;
-        return;
-      }
-
-      const id = $(elem)
-        .find("a")
-        .first()
-        .attr("href");
-      const cijena = $(elem)
-        .find(".cijena > .datum > span")
-        .first()
-        .text();
-      const image = $(elem)
-        .find("a > .slika > img")
-        .first()
-        .attr("src");
-
-      items.push({ url: id, price: cijena, image, date: itemDate });
-    });
-  return items;
-}
-
-module.exports = scrapTheItems;

app/lib/sendNotification.js

@@ -1,58 +0,0 @@
-const scrapTheItems = require("./scrapTheItems");
-const convertToDate = require("./convertToDate");
-const AWS = require('aws-sdk');
-// AWS.config.update({region: 'eu-central-1'});
-
-
-async function sendNotification(marketAlert) {
-  const { id, email, olx_url } = marketAlert;
-  let url =
-    "https://www.olx.ba/pretraga?" + olx_url + "&sort_order=desc&sort_po=datum";
-  let newItems = await scrapTheItems(url);
-  let lastDate = newItems.length && newItems[0].date;
-  let message =
-    newItems.length &&
-    newItems.reduce(
-      (mes, item) => mes + `<strong>${item.url} i  ${item.price}</strong>`,
-      ""
-    );
-
-  // Create sendEmail params
-  const params = {
-    Destination: { /* required */
-      CcAddresses: [
-      ],
-      ToAddresses: [
-        email
-      ]
-    },
-    Message: { /* required */
-      Body: { /* required */
-        Html: {
-         Charset: "UTF-8",
-         Data: message
-        },
-        Text: {
-         Charset: "UTF-8",
-         Data: message // TODO: convert to text
-        }
-       },
-     Subject: {
-      Charset: 'UTF-8',
-      Data: 'Javimi alert'
-     }
-    },
-    Source: 'info@saburly.com', /* required */
-    ReplyToAddresses: [
-       'info@saburly.com',
-    ],
-  };
-
-  if (message) {
-    const sendPromise = new AWS.SES({apiVersion: '2010-12-01'}).sendEmail(params).promise();
-    await sendPromise;
-    return { id, date: String(convertToDate(lastDate)) };
-  }
-}
-
-module.exports = sendNotification;

app/migrations/20190417035319-create-market-alert.js

@@ -1,7 +1,7 @@
-'use strict';
+"use strict";
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('MarketAlerts', {
+    return queryInterface.createTable("MarketAlerts", {
       id: {
         allowNull: false,
         autoIncrement: true,
@@ -29,6 +29,6 @@ module.exports = {
     });
   },
   down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('MarketAlerts');
+    return queryInterface.dropTable("MarketAlerts");
   }
 };

app/migrations/20190417035707-create-real-estate-request.js

@@ -1,7 +1,7 @@
-'use strict';
+"use strict";
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('RealEstateRequests', {
+    return queryInterface.createTable("RealEstateRequests", {
       id: {
         allowNull: false,
         autoIncrement: true,
@@ -28,6 +28,6 @@ module.exports = {
     });
   },
   down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('RealEstateRequests');
+    return queryInterface.dropTable("RealEstateRequests");
   }
 };

app/migrations/20190427043621-add_city_to_real_estate_request.js

@@ -1,18 +1,15 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.addColumn(
-      'RealEstateRequests',
-      'city',
-     Sequelize.STRING
+      "RealEstateRequests",
+      "city",
+      Sequelize.STRING
     );
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-      'RealEstateRequests',
-      'city'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "city");
   }
 };

app/migrations/20190429050222-add_place_to_real_estate_request.js

@@ -1,18 +1,15 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.addColumn(
-      'RealEstateRequests',
-      'place',
-     Sequelize.STRING
+      "RealEstateRequests",
+      "place",
+      Sequelize.STRING
     );
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-      'RealEstateRequests',
-      'place'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "place");
   }
 };

app/migrations/20190516180226-rename-place-column.js

@@ -1,19 +1,19 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.renameColumn(
-        'RealEstateRequests',
-        'place',
-        'municipality'
+      "RealEstateRequests",
+      "place",
+      "municipality"
     );
   },
 
   down: (queryInterface, Sequelize) => {
     return queryInterface.renameColumn(
-        'RealEstateRequests',
-        'municipality',
-        'place'
+      "RealEstateRequests",
+      "municipality",
+      "place"
     );
   }
 };

app/migrations/20190516222240-rename-city-column.js

@@ -1,19 +1,11 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.renameColumn(
-        'RealEstateRequests',
-        'city',
-        'region'
-    );
+    return queryInterface.renameColumn("RealEstateRequests", "city", "region");
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.renameColumn(
-        'RealEstateRequests',
-        'region',
-        'city'
-    );
+    return queryInterface.renameColumn("RealEstateRequests", "region", "city");
   }
 };

app/migrations/20190517072957-add-size-to-real-estate-request.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'RealEstateRequests',
-        'size',
-        {
-          type: Sequelize.STRING
-        }
-    );
+    return queryInterface.addColumn("RealEstateRequests", "size", {
+      type: Sequelize.STRING
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'RealEstateRequests',
-        'size'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "size");
   }
 };

app/migrations/20190517090015-add-gardenSize-to-real-estate-request.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'RealEstateRequests',
-        'gardenSize',
-        {
-          type: Sequelize.STRING
-        }
-    );
+    return queryInterface.addColumn("RealEstateRequests", "gardenSize", {
+      type: Sequelize.STRING
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'RealEstateRequests',
-        'gardenSize'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "gardenSize");
   }
 };

app/migrations/20190517092716-add-price-to-real-estate-request.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'RealEstateRequests',
-        'price',
-        {
-          type: Sequelize.STRING
-        }
-    );
+    return queryInterface.addColumn("RealEstateRequests", "price", {
+      type: Sequelize.STRING
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'RealEstateRequests',
-        'price'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "price");
   }
 };

app/migrations/20190523144812-activate-postgis.js

@@ -1,15 +1,19 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.sequelize.query("CREATE EXTENSION postgis").then(([results, metadata]) => {
-       /// No result
-      })
+    return queryInterface.sequelize
+      .query("CREATE EXTENSION postgis")
+      .then(([results, metadata]) => {
+        /// No result
+      });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.sequelize.query("DROP EXTENSION IF EXISTS postgis").then(([results, metadata]) => {
+    return queryInterface.sequelize
+      .query("DROP EXTENSION IF EXISTS postgis")
+      .then(([results, metadata]) => {
         /// No result
-       })
+      });
   }
 };

app/migrations/20190523151420-add-bounding-box-column.js

@@ -1,17 +1,21 @@
-'use strict';
+"use strict";
 
 module.exports = {
-
   up: (queryInterface, Sequelize) => {
-
-    return queryInterface.sequelize.query("ALTER TABLE \"RealEstateRequests\" ADD COLUMN bounding_box geometry(Polygon);").then(([results, metadata]) => {
-      /// No result
-    })
+    return queryInterface.sequelize
+      .query(
+        'ALTER TABLE "RealEstateRequests" ADD COLUMN bounding_box geometry(Polygon);'
+      )
+      .then(([results, metadata]) => {
+        /// No result
+      });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.sequelize.query("ALTER TABLE \"RealEstateRequests\" DROP COLUMN bounding_box").then(([results, metadata]) => {
-      /// No result
-    })
+    return queryInterface.sequelize
+      .query('ALTER TABLE "RealEstateRequests" DROP COLUMN bounding_box')
+      .then(([results, metadata]) => {
+        /// No result
+      });
   }
-};
+};

app/migrations/20190529093410-slider-fields.js

@@ -1,27 +1,48 @@
 module.exports = {
-    up: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.addColumn('RealEstateRequests', 'sizeRange', {
-                    type: Sequelize.STRING
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'gardenSizeRange', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'priceRange', {
-                    type: Sequelize.STRING,
-                }, { transaction: t })
-            ])
-        })
-    },
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
 
-    down: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('RealEstateRequests', 'sizeRange', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'gardenSizeRange', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'priceRange', { transaction: t })
-            ])
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceRange", {
+          transaction: t
         })
-    }
-};
+      ]);
+    });
+  }
+};

app/migrations/20190530101945-range-fields.js

@@ -1,63 +1,147 @@
 module.exports = {
-    up: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('RealEstateRequests', 'sizeRange', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'gardenSizeRange', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'priceRange', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'size', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'gardenSize', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'price', { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'gardenSizeMin', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'gardenSizeMax', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'sizeMin', {
-                    type: Sequelize.INTEGER
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'sizeMax', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'priceMin', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'priceMax', {
-                    type: Sequelize.INTEGER
-                }, { transaction: t })
-            ])
-        })
-    },
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "sizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceRange", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "size", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSize", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "price", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "sizeMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMin",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMax",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
 
-    down: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('RealEstateRequests', 'gardenSizeMin', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'gardenSizeMax', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'sizeMin', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'sizeMax', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'priceMin', { transaction: t }),
-                queryInterface.removeColumn('RealEstateRequests', 'priceMin', { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'priceMax', {
-                    type: Sequelize.STRING
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'gardenSizeRange', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'priceRange', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'size', {
-                    type: Sequelize.STRING
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'gardenSize', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('RealEstateRequests', 'price', {
-                    type: Sequelize.STRING,
-                }, { transaction: t })
-            ])
-        })
-    }
-};
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "gardenSizeMax", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "sizeMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "sizeMax", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceMin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("RealEstateRequests", "priceMin", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceMax",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSizeRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "priceRange",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "size",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "gardenSize",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "RealEstateRequests",
+          "price",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  }
+};

app/migrations/20190531111232-subscribed-boolean.js

@@ -1,18 +1,15 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
     return queryInterface.addColumn(
-      'RealEstateRequests',
-      'subscribed',
-     Sequelize.BOOLEAN
+      "RealEstateRequests",
+      "subscribed",
+      Sequelize.BOOLEAN
     );
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-      'RealEstateRequests',
-      'subscribed'
-    );
+    return queryInterface.removeColumn("RealEstateRequests", "subscribed");
   }
 };

app/migrations/20190618103020-expand-maketalert.js

@@ -1,37 +1,70 @@
-'use strict';
+"use strict";
 
 module.exports = {
-    up: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.addColumn('MarketAlerts', 'size', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'gardenSize', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'price', {
-                    type: Sequelize.INTEGER,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'municipality', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'region', {
-                    type: Sequelize.STRING,
-                }, { transaction: t })
-            ])
-        })
-    },
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "size",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "gardenSize",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "price",
+          {
+            type: Sequelize.INTEGER
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "municipality",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "region",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
 
-    down: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('MarketAlerts', 'size', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'gardenSize', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'price', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'municipality', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'region', { transaction: t })
-            ])
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "size", { transaction: t }),
+        queryInterface.removeColumn("MarketAlerts", "gardenSize", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "price", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "municipality", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "region", {
+          transaction: t
         })
-    }
+      ]);
+    });
+  }
 };

app/migrations/20190618124522-marketalerts-additional-info.js

@@ -1,33 +1,59 @@
-'use strict';
+"use strict";
 
 module.exports = {
-    up: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('MarketAlerts', 'olxUrl', { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'url', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'realestateOrigin', {
-                    type: Sequelize.STRING,
-                }, { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'originId', {
-                    type: Sequelize.STRING,
-                }, { transaction: t })
-            ])
-        })
-    },
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "olxUrl", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "url",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "realestateOrigin",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        ),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "originId",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  },
 
-    down: (queryInterface, Sequelize) => {
-        return queryInterface.sequelize.transaction((t) => {
-            return Promise.all([
-                queryInterface.removeColumn('MarketAlerts', 'url', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'realestateOrigin', { transaction: t }),
-                queryInterface.removeColumn('MarketAlerts', 'originId', { transaction: t }),
-                queryInterface.addColumn('MarketAlerts', 'olxUrl', {
-                    type: Sequelize.STRING
-                }, { transaction: t })
-            ])
-        })
-    }
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.transaction(t => {
+      return Promise.all([
+        queryInterface.removeColumn("MarketAlerts", "url", { transaction: t }),
+        queryInterface.removeColumn("MarketAlerts", "realestateOrigin", {
+          transaction: t
+        }),
+        queryInterface.removeColumn("MarketAlerts", "originId", {
+          transaction: t
+        }),
+        queryInterface.addColumn(
+          "MarketAlerts",
+          "olxUrl",
+          {
+            type: Sequelize.STRING
+          },
+          { transaction: t }
+        )
+      ]);
+    });
+  }
 };

app/migrations/20190621162321-add-category-to-marketalert.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'realEstateType',
-        {
-          type: Sequelize.STRING
-        }
-    );
+    return queryInterface.addColumn("MarketAlerts", "realEstateType", {
+      type: Sequelize.STRING
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'realEstateType'
-    );
+    return queryInterface.removeColumn("MarketAlerts", "realEstateType");
   }
 };

app/migrations/20190625120813-add-notification-sent-boolean-marketalerts.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'notified',
-        {
-          type: Sequelize.BOOLEAN
-        }
-    );
+    return queryInterface.addColumn("MarketAlerts", "notified", {
+      type: Sequelize.BOOLEAN
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'notified'
-    );
+    return queryInterface.removeColumn("MarketAlerts", "notified");
   }
 };

app/migrations/20190628165512-add-title-to-marketalerts.js

@@ -1,20 +1,13 @@
-'use strict';
+"use strict";
 
 module.exports = {
   up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn(
-        'MarketAlerts',
-        'title',
-        {
-          type: Sequelize.STRING
-        }
-    );
+    return queryInterface.addColumn("MarketAlerts", "title", {
+      type: Sequelize.STRING
+    });
   },
 
   down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn(
-        'MarketAlerts',
-        'title'
-    );
+    return queryInterface.removeColumn("MarketAlerts", "title");
   }
 };

app/migrations/20190702132143-add-RREquet-uuid-to-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "request", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "request");
+  }
+};

app/migrations/20190710141356-add-has-location-to-marketalerts.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("MarketAlerts", "hasLocation", {
+      type: Sequelize.BOOLEAN
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("MarketAlerts", "hasLocation");
+  }
+};

app/migrations/20190909110623-add-location-input-to-real-estate-request.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstateRequests", "locationInput", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstateRequests", "locationInput");
+  }
+};

app/migrations/20190910033923-change-bounding-box-from-snake-case-to-camel-case.js

@@ -0,0 +1,19 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "bounding_box",
+      "boundingBox"
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.renameColumn(
+      "RealEstateRequests",
+      "boundingBox",
+      "bounding_box"
+    );
+  }
+};

app/migrations/20190912191829-add-realEstates-table.js

@@ -0,0 +1,72 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      url: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      agencyObjectId: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      originAgencyName: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      realEstateType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      adType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      price: Sequelize.REAL,
+      area: Sequelize.REAL,
+      gardenSize: Sequelize.REAL,
+      streetNumber: Sequelize.INTEGER,
+      streetName: Sequelize.TEXT,
+      locality: Sequelize.TEXT,
+      municipality: Sequelize.TEXT,
+      city: Sequelize.TEXT,
+      region: Sequelize.TEXT,
+      entity: Sequelize.TEXT,
+      country: Sequelize.TEXT,
+      locationLat: Sequelize.REAL,
+      locationLong: Sequelize.REAL,
+      lastTimeCrawled: {
+        type: Sequelize.DATE,
+        allowNull: false
+      },
+      deleted: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false
+      },
+      sold: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("RealEstates", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("RealEstates", {});
+  }
+};

app/migrations/20190912215313-add-searchRequests-table.js

@@ -0,0 +1,79 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.UUID,
+        defaultValue: Sequelize.UUIDV4,
+        allowNull: false,
+        primaryKey: true
+      },
+      areaToSearch: {
+        type: Sequelize.GEOMETRY("POLYGON", 4326),
+        allowNull: false,
+        defaultValue: {
+          type: "Polygon",
+          coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+          crs: { type: "name", properties: { name: "EPSG:4326" } }
+        }
+      },
+      realEstateType: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      adType: {
+        type: Sequelize.TEXT,
+        allowNull: false,
+        defaultValue: "sell"
+      },
+      email: Sequelize.TEXT,
+      locality: Sequelize.TEXT,
+      municipality: Sequelize.TEXT,
+      city: Sequelize.TEXT,
+      region: Sequelize.TEXT,
+      entity: Sequelize.TEXT,
+      country: Sequelize.TEXT,
+      sizeMin: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      sizeMax: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      priceMin: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      priceMax: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        defaultValue: 0
+      },
+      gardenSizeMin: Sequelize.INTEGER,
+      gardenSizeMax: Sequelize.INTEGER,
+      subscribed: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false,
+        defaultValue: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("SearchRequests", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("SearchRequests", {});
+  }
+};

app/migrations/20190912215556-add-searchRequestMatches-table.js

@@ -0,0 +1,53 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false
+      },
+      searchRequestId: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "SearchRequests",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        primaryKey: true,
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      notified: {
+        type: Sequelize.BOOLEAN,
+        allowNull: false,
+        defaultValue: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("SearchRequestMatches", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("SearchRequestMatches", {});
+  }
+};

app/migrations/20190916160603-add-title-column-to-realEstates.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "title", {
+      type: Sequelize.STRING
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "title");
+  }
+};

app/migrations/20190916230229-add-description-columns-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "shortDescription", {
+        type: Sequelize.STRING
+      }),
+      queryInterface.addColumn("RealEstates", "longDescription", {
+        type: Sequelize.STRING
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "shortDescription"),
+      queryInterface.removeColumn("RealEstates", "longDescription")
+    ]);
+  }
+};

app/migrations/20190916231241-add-status-column-to-realEstates.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "adStatus", {
+      type: Sequelize.INTEGER
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "adStatus");
+  }
+};

app/migrations/20190917090318-add-composite-key-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addConstraint(
+      "RealEstates",
+      ["originAgencyName", "agencyObjectId"],
+      {
+        type: "unique",
+        name: "agencyNameObjectIdUniqueKey"
+      }
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeConstraint(
+      "RealEstates",
+      "agencyNameObjectIdUniqueKey"
+    );
+  }
+};

app/migrations/20190917101052-remove-lastTimeCrawled-column-from-realEstates.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("RealEstates", "lastTimeCrawled");
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("RealEstates", "lastTimeCrawled", {
+      type: Sequelize.DATE,
+      notNull: true
+    });
+  }
+};

app/migrations/20190917102437-remove-deleted-sold-columns-from-realEstates.js

@@ -0,0 +1,23 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "deleted"),
+      queryInterface.removeColumn("RealEstates", "sold")
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "deleted", {
+        type: Sequelize.BOOLEAN,
+        notNull: true
+      }),
+      queryInterface.addColumn("RealEstates", "sold", {
+        type: Sequelize.BOOLEAN,
+        notNull: true
+      })
+    ]);
+  }
+};

app/migrations/20190917115804-change-title-short-long-description-columns-in-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.changeColumn("RealEstates", "shortDescription", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.changeColumn("RealEstates", "longDescription", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.changeColumn("RealEstates", "title", {
+        type: Sequelize.TEXT
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([]);
+  }
+};

app/migrations/20190923185802-add-published-renewed-dates-to-realEstates.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "publishedDate", {
+        type: Sequelize.DATE
+      }),
+      queryInterface.addColumn("RealEstates", "renewedDate", {
+        type: Sequelize.DATE
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "renewedDate"),
+      queryInterface.removeColumn("RealEstates", "publishedDate")
+    ]);
+  }
+};

app/migrations/20190927121449-change-adType-in-searchRequests-table-to-SALE.js

@@ -0,0 +1,15 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'SALE' WHERE "adType" = 'sell';`
+    );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize.query(
+      `UPDATE "SearchRequests" SET "adType" = 'sell' WHERE "adType" = 'SALE';`
+    );
+  }
+};

app/migrations/20190927180101-change-change-realEstateType-in-searchRequests-table.js

@@ -0,0 +1,31 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'HOUSE' WHERE "realEstateType" = 'kuca';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'FLAT' WHERE "realEstateType" = 'stan';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'COTTAGE' WHERE "realEstateType" = 'vikendica';`
+      )
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'kuca' WHERE "realEstateType" = 'HOUSE';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'stan' WHERE "realEstateType" = 'FLAT';`
+      ),
+      queryInterface.sequelize.query(
+        `UPDATE "SearchRequests" SET "realEstateType" = 'vikendica' WHERE "realEstateType" = 'COTTAGE';`
+      )
+    ]);
+  }
+};

app/migrations/20191101113541-add-emailFrequency-column-to-searchRequest-table.js

@@ -0,0 +1,15 @@
+"use strict";
+const { EMAIL_FREQUENCY } = require("../common/enums");
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "emailFrequency", {
+      type: Sequelize.TEXT,
+      defaultValue: EMAIL_FREQUENCY.ASAP.stringId
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "emailFrequency");
+  }
+};

app/migrations/20191105111202-add-deletedEmail-column-to-searchRequest-table.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "deletedEmail", {
+      type: Sequelize.TEXT
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "deletedEmail");
+  }
+};

app/migrations/20191105174319-add-additional-fields-to-realEstates-table.js

@@ -0,0 +1,163 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("RealEstates", "numberOfRooms", {
+        type: Sequelize.REAL
+      }),
+      queryInterface.addColumn("RealEstates", "numberOfFloors", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("RealEstates", "floor", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("RealEstates", "accessRoadType", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.addColumn("RealEstates", "heatingType", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.addColumn("RealEstates", "furnishingType", {
+        type: Sequelize.TEXT
+      }),
+      queryInterface.addColumn("RealEstates", "balcony", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "newBuilding", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "elevator", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "water", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "electricity", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "drainageSystem", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "registeredInZkBooks", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "recentlyAdapted", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "parking", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "garage", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "gas", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "antiTheftDoor", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "airCondition", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "phoneConnection", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "cableTV", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "internet", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "basementAttic", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "storeRoom", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "videoSurveillance", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "alarm", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "suitableForStudents", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "includingBills", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "animalsAllowed", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "pool", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "exchange", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "urbanPlanPermit", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "buildingPermit", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "utilityConnection", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("RealEstates", "distanceToRiver", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("RealEstates", "numberOfViewsAgency", {
+        type: Sequelize.INTEGER,
+        defaultValue: 0
+      }),
+      queryInterface.addColumn("RealEstates", "numberOfViewsKivi", {
+        type: Sequelize.INTEGER,
+        defaultValue: 0
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("RealEstates", "numberOfRooms"),
+      queryInterface.removeColumn("RealEstates", "numberOfFloors"),
+      queryInterface.removeColumn("RealEstates", "floor"),
+      queryInterface.removeColumn("RealEstates", "accessRoadType"),
+      queryInterface.removeColumn("RealEstates", "heatingType"),
+      queryInterface.removeColumn("RealEstates", "furnishingType"),
+      queryInterface.removeColumn("RealEstates", "balcony"),
+      queryInterface.removeColumn("RealEstates", "newBuilding"),
+      queryInterface.removeColumn("RealEstates", "elevator"),
+      queryInterface.removeColumn("RealEstates", "water"),
+      queryInterface.removeColumn("RealEstates", "electricity"),
+      queryInterface.removeColumn("RealEstates", "drainageSystem"),
+      queryInterface.removeColumn("RealEstates", "registeredInZkBooks"),
+      queryInterface.removeColumn("RealEstates", "recentlyAdapted"),
+      queryInterface.removeColumn("RealEstates", "parking"),
+      queryInterface.removeColumn("RealEstates", "garage"),
+      queryInterface.removeColumn("RealEstates", "gas"),
+      queryInterface.removeColumn("RealEstates", "antiTheftDoor"),
+      queryInterface.removeColumn("RealEstates", "airCondition"),
+      queryInterface.removeColumn("RealEstates", "phoneConnection"),
+      queryInterface.removeColumn("RealEstates", "cableTV"),
+      queryInterface.removeColumn("RealEstates", "internet"),
+      queryInterface.removeColumn("RealEstates", "basementAttic"),
+      queryInterface.removeColumn("RealEstates", "storeRoom"),
+      queryInterface.removeColumn("RealEstates", "videoSurveillance"),
+      queryInterface.removeColumn("RealEstates", "alarm"),
+      queryInterface.removeColumn("RealEstates", "suitableForStudents"),
+      queryInterface.removeColumn("RealEstates", "includingBills"),
+      queryInterface.removeColumn("RealEstates", "animalsAllowed"),
+      queryInterface.removeColumn("RealEstates", "pool"),
+      queryInterface.removeColumn("RealEstates", "exchange"),
+      queryInterface.removeColumn("RealEstates", "urbanPlanPermit"),
+      queryInterface.removeColumn("RealEstates", "buildingPermit"),
+      queryInterface.removeColumn("RealEstates", "utilityConnection"),
+      queryInterface.removeColumn("RealEstates", "distanceToRiver"),
+      queryInterface.removeColumn("RealEstates", "numberOfViewsAgency"),
+      queryInterface.removeColumn("RealEstates", "numberOfViewsKivi")
+    ]);
+  }
+};

app/migrations/20191118105541-add-additional-fields-to-searchRequests-table.js

@@ -0,0 +1,64 @@
+"use strict";
+const { ACCESS_ROAD_TYPE, HEATING_TYPE } = require("../common/enums");
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.addColumn("SearchRequests", "includeIncompleteAds", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("SearchRequests", "balcony", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("SearchRequests", "newBuilding", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("SearchRequests", "elevator", {
+        type: Sequelize.BOOLEAN
+      }),
+      queryInterface.addColumn("SearchRequests", "numberOfRoomsMin", {
+        type: Sequelize.REAL
+      }),
+      queryInterface.addColumn("SearchRequests", "numberOfRoomsMax", {
+        type: Sequelize.REAL
+      }),
+      queryInterface.addColumn("SearchRequests", "numberOfFloorsMin", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("SearchRequests", "numberOfFloorsMax", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("SearchRequests", "floorMin", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("SearchRequests", "floorMax", {
+        type: Sequelize.INTEGER
+      }),
+      queryInterface.addColumn("SearchRequests", "accessRoadType", {
+        type: Sequelize.TEXT,
+        defaultValue: ACCESS_ROAD_TYPE.ANY.id
+      }),
+      queryInterface.addColumn("SearchRequests", "heatingType", {
+        type: Sequelize.TEXT,
+        defaultValue: HEATING_TYPE.ANY.id
+      })
+    ]);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return Promise.all([
+      queryInterface.removeColumn("SearchRequests", "includeIncompleteAds"),
+      queryInterface.removeColumn("SearchRequests", "balcony"),
+      queryInterface.removeColumn("SearchRequests", "newBuilding"),
+      queryInterface.removeColumn("SearchRequests", "elevator"),
+      queryInterface.removeColumn("SearchRequests", "numberOfRoomsMin"),
+      queryInterface.removeColumn("SearchRequests", "numberOfRoomsMax"),
+      queryInterface.removeColumn("SearchRequests", "numberOfFloorsMin"),
+      queryInterface.removeColumn("SearchRequests", "numberOfFloorsMax"),
+      queryInterface.removeColumn("SearchRequests", "floorMin"),
+      queryInterface.removeColumn("SearchRequests", "floorMax"),
+      queryInterface.removeColumn("SearchRequests", "accessRoadType"),
+      queryInterface.removeColumn("SearchRequests", "heatingType")
+    ]);
+  }
+};

app/migrations/20200121000524-add-priceHistory-table.js

@@ -0,0 +1,42 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: Sequelize.REAL,
+        unique: "uniquePriceRealEstate"
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("PriceHistory", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("PriceHistory", {});
+  }
+};

app/migrations/20200121094500-add-constraint-priceHistory.js

@@ -0,0 +1,10 @@
+"use strict";
+module.exports = {
+  up: (queryInterface, Sequelize) =>
+    queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], {
+      type: "unique",
+      name: "uniquePriceRealEstate"
+    }),
+  down: queryInterface =>
+    queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate")
+};

app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "includeWithoutPrice", {
+      type: Sequelize.BOOLEAN,
+      defaultValue: true
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "includeWithoutPrice");
+  }
+};

app/models/index.js

@@ -1,27 +1,39 @@
-'use strict';
+"use strict";
 
-const fs = require('fs');
-const path = require('path');
-const Sequelize = require('sequelize');
+const fs = require("fs");
+const path = require("path");
+const Sequelize = require("sequelize");
 const basename = path.basename(__filename);
-const env = process.env.NODE_ENV || 'development';
-const config = require(__dirname + '/../config/config.json')[env];
+const env = process.env.NODE_ENV || "development";
+const config = require(__dirname + "/../config/config.json")[env];
 const db = {};
 
+config.username = process.env.DB_USERNAME || config.username;
+config.password = process.env.DB_PASSWORD || config.password;
+config.database = process.env.DB_NAME || config.database;
+config.port = process.env.DB_PORT || config.port;
+config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;
+
 let sequelize;
 if (config.use_env_variable) {
   sequelize = new Sequelize(process.env[config.use_env_variable], config);
 } else {
-  sequelize = new Sequelize(config.database, config.username, config.password, config);
+  sequelize = new Sequelize(
+    config.database,
+    config.username,
+    config.password,
+    config
+  );
 }
 
-fs
-  .readdirSync(__dirname)
+fs.readdirSync(__dirname)
   .filter(file => {
-    return (file.indexOf('.') !== 0) && (file !== basename) && (file.slice(-3) === '.js');
+    return (
+      file.indexOf(".") !== 0 && file !== basename && file.slice(-3) === ".js"
+    );
   })
   .forEach(file => {
-    const model = sequelize['import'](path.join(__dirname, file));
+    const model = sequelize["import"](path.join(__dirname, file));
     db[model.name] = model;
   });
 

app/models/marketalert.js

@@ -1,26 +0,0 @@
-'use strict';
-module.exports = (sequelize, DataTypes) => {
-  const MarketAlert = sequelize.define('MarketAlert', {
-    url: DataTypes.STRING,
-    realestateOrigin: DataTypes.STRING,
-    originId: DataTypes.STRING,
-    lastDate: DataTypes.STRING,
-    size : DataTypes.INTEGER,
-    gardenSize : DataTypes.INTEGER,
-    price : DataTypes.INTEGER,
-    municipality : DataTypes.STRING,
-    region : DataTypes.STRING,
-    realEstateType : DataTypes.STRING,
-    notified : DataTypes.BOOLEAN,
-    title : DataTypes.STRING,
-    
-    email: { 
-      type: DataTypes.STRING,
-      allowNul: false
-    }
-  }, {});
-  MarketAlert.associate = function(models) {
-    // associations can be defined here
-  };
-  return MarketAlert;
-};