Add ssl to sequelize

Improve olx scrapper

See merge request saburly/marketalarm/web!113

app/common/enums.js

@@ -304,7 +304,6 @@ const AD_AGENCY = {
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
   AKTIDO: "AKTIDO",
-  KIVI: "KIVI",
   SALJIC: "SALJIC"
 };
 

app/common/publishEnums.js

@@ -1,492 +0,0 @@
-const {
-  AD_CATEGORY,
-  ACCESS_ROAD_TYPE,
-  HEATING_TYPE,
-  FURNISHING_TYPE
-} = require("./enums");
-
-const BASIC_BOOLEAN_PUBLISH = [
-  {
-    dbField: "newBuilding",
-    title: "Novogradnja",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "balcony",
-    title: "Balkon",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-
-  {
-    dbField: "elevator",
-    title: "Lift",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "recentlyAdapted",
-    title: "Nedavno adaptirano",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-const BASIC_INPUT_PUBLISH = [
-  {
-    dbField: "title",
-    title: "Naslov",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["required"]
-  },
-  {
-    dbField: "shortDescription",
-    title: "Opis",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: []
-  },
-  {
-    dbField: "price",
-    title: "Cijena (KM)",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "area",
-    title: "Površina (m\xB2)",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "gardenSize",
-    title: "Površina okućnice (m\xB2)",
-    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "streetName",
-    title: "Adresa",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: []
-  },
-  {
-    dbField: "numberOfRooms",
-    title: "Broj soba",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ],
-    constraint: ["integer"]
-  },
-  {
-    dbField: "numberOfFloors",
-    title: "Broj spratova",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ],
-    constraint: ["integer"]
-  },
-  {
-    dbField: "floor",
-    title: "Sprat",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.OFFICE
-    ],
-    constraint: ["integer"]
-  }
-];
-
-const BASIC_SEGMENT_PUBLISH = [
-  {
-    dbField: "furnishingType",
-    title: "Namještaj",
-    values: Object.keys(FURNISHING_TYPE).map(key => FURNISHING_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-const ADDITIONAL_BOOLEAN_PUBLISH = [
-  {
-    dbField: "water",
-    title: "Voda",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "electricity",
-    title: "Struja",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "drainageSystem",
-    title: "Kanalizacija",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "registeredInZkBooks",
-    title: "Uknjiženo",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-
-  {
-    dbField: "parking",
-    title: "Parking",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "garage",
-    title: "Garaža",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "gas",
-    title: "Plin",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "antiTheftDoor",
-    title: "Blindirana vrata",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "airCondition",
-    title: "Klimatizirano",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "phoneConnection",
-    title: "Telefon",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "cableTV",
-    title: "Kablovska",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "internet",
-    title: "Internet",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "basementAttic",
-    title: "Podrum-Tavan",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "storeRoom",
-    title: "Ostava",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "videoSurveillance",
-    title: "Video nadzor",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "alarm",
-    title: "Alarm",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "suitableForStudents",
-    title: "Za studente",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "includingBills",
-    title: "Uključen trošak režija",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "animalsAllowed",
-    title: "Kućni ljubimci dozvoljeni",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "pool",
-    title: "Bazen",
-    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE]
-  },
-  {
-    dbField: "exchange",
-    title: "Zamjena",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "urbanPlanPermit",
-    title: "Urbanistička dozvola",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "buildingPermit",
-    title: "Građevinska dozvola",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  }
-];
-
-const ADDITIONAL_INPUT_PUBLISH = [
-  {
-    dbField: "longDescription",
-    title: "Detaljan opis",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  }
-];
-
-const ADDITIONAL_SEGMENT_PUBLISH = [
-  {
-    dbField: "accessRoadType",
-    title: "Pristupni put",
-    values: Object.keys(ACCESS_ROAD_TYPE).map(key => ACCESS_ROAD_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "heatingType",
-    title: "Grijanje",
-    values: Object.keys(HEATING_TYPE).map(key => HEATING_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-module.exports = {
-  BASIC_INPUT_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  BASIC_BOOLEAN_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH
-};

app/config/appConfig.js

@@ -9,7 +9,7 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const STAGING = process.env.SETTINGS !== "production";
+const STAGING = process.env.ENVIRONMENT !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
@@ -34,13 +34,22 @@ const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
 
 const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
-const GOOGLE_MAP_KEY = process.env.GOOGLE_MAP_KEY || "";
+const API_MAP_KEY = process.env.API_MAP_KEY || "";
 
 const PROSTOR_LOGIN = {
   EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
   PASSWORD: process.env.PROSTOR_LOGIN_PASS
 };
 
+const USER_AGENT =
+  process.env.USER_AGENT ||
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
+
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
+const NODE_FETCH_TIMEOUT_MS = parseInt(process.env.NODE_FETCH_TIMEOUT_MS) || 60000
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -51,8 +60,13 @@ module.exports = {
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
   PRINT_CRAWLER_DEBUG,
-  GOOGLE_MAP_KEY,
+  API_MAP_KEY,
   STAGING,
   CHECK_UP_DAYS,
-  PROSTOR_LOGIN
+  PROSTOR_LOGIN,
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL,
+  NODE_FETCH_TIMEOUT_MS
 };

app/controllers/deleteRealEstate.js

@@ -1,34 +0,0 @@
-const { currentKiviRealEstate } = require("../helpers/url");
-const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
-
-const getDeletePublishedAd = async (req, res) => {
-  const title = "Uspješno ste izbrisali svoj oglas iz baze.";
-
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  if (!kiviOriginal || !kiviOriginal.kiviAdId) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  realEstate.deleted = true;
-  realEstate.deletedAt = Date.now();
-
-  kiviOriginal.email = "";
-
-  await realEstate.save();
-
-  await kiviOriginal.save();
-
-  res.render("deleteRealEstate", { nextStep: "/", title });
-};
-
-module.exports = {
-  getDeletePublishedAd
-};

app/controllers/editSuccess.js

@@ -1,8 +0,0 @@
-const editSuccess = async (req, res) => {
-  const title = "Uspjeh!";
-  res.render("editSuccess", { title });
-};
-
-module.exports = {
-  editSuccess
-};

app/controllers/publishRealEstate.js

@@ -1,385 +0,0 @@
-const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
-const {
-  bulkUpsertKiviPhotos,
-  findPhotosForKiviAd,
-  deleteUrlPhotosAfterUpdate
-} = require("../helpers/db/kiviOriginalAdsPhotos");
-const { currentKiviRealEstate } = require("../helpers/url");
-const {
-  notifyForNewRealEstates,
-  notifyForNewAdPublish
-} = require("../services/notificationService");
-const validate = require("validate.js");
-
-const {
-  AD_CATEGORY,
-  FURNISHING_TYPE,
-  ACCESS_ROAD_TYPE,
-  HEATING_TYPE
-} = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
-
-const {
-  BASIC_BOOLEAN_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH,
-  BASIC_INPUT_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH
-} = require("../common/publishEnums");
-
-const getPublishInputs = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(
-    kiviOriginal.dataValues.kiviAdId
-  );
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const pageTitle = "Podaci o nekretnini";
-
-  const {
-    price,
-    area,
-    adType,
-    realEstateType,
-    locationLat,
-    locationLong,
-    accessRoadType,
-    heatingType,
-    balcony,
-    newBuilding,
-    elevator,
-    recentlyAdapted,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit,
-    furnishingType,
-    shortDescription,
-    streetName,
-    title,
-    longDescription
-  } = realEstate;
-
-  const email = kiviOriginal.dataValues.email;
-  //If email is not empty - has string value, then proces of publishing has been conducted alredy
-  //That means user is editing existing real estate ad not publishing new one
-  let editingRealEstate = false;
-  if (email) {
-    editingRealEstate = true;
-  }
-  //If we are editing real estate ad we need to fetch and show images on server
-  const urlGooglePrefix =
-    "https://storage.cloud.google.com/marketalarm-photos/";
-  let realEstatePhotosUrls = [];
-  if (editingRealEstate) {
-    const realEstatePhotosData = await findPhotosForKiviAd(
-      kiviOriginal.dataValues.kiviAdId
-    );
-    realEstatePhotosData.map(row => {
-      realEstatePhotosUrls.push(urlGooglePrefix + row.dataValues.photoUrl);
-    });
-  } else {
-    realEstatePhotosUrls.push(false);
-  }
-
-  const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
-
-  // TODO: Maybe this is slow, pay attention to this
-  const filterInputs = filterObject => {
-    const filterCategories = filterObject.categoriesToShow;
-    return filterCategories.indexOf(category) !== -1;
-  };
-  //Boolean inputs to be shown on Basic Data tab
-  const basicBooleanPublishInputs = BASIC_BOOLEAN_PUBLISH.filter(filterInputs);
-  const basicBooleanPublishValues = {
-    balcony,
-    elevator,
-    newBuilding,
-    recentlyAdapted
-  };
-  //Boolean inputs to be shown on Additional Data tab
-  const additionalBooleanPublishInputs = ADDITIONAL_BOOLEAN_PUBLISH.filter(
-    filterInputs
-  );
-  const additionalBooleanPublishValues = {
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit
-  };
-  //Segment select inputs to be shown on Basic Data tab
-  const basicSegmentSelectInputs = BASIC_SEGMENT_PUBLISH.filter(filterInputs);
-  const basicSegmentSelectValues = {
-    furnishingType
-  };
-  //Segment select inputs to be shown on Additional Data tab
-  const additionalSegmentSelectInputs = ADDITIONAL_SEGMENT_PUBLISH.filter(
-    filterInputs
-  );
-  const additionalSegmentSelectValues = {
-    accessRoadType,
-    heatingType
-  };
-  //Input text type inputs to be shown on Basic Data tab
-  const basicInputInputs = BASIC_INPUT_PUBLISH.filter(filterInputs);
-  const basicInputValues = {
-    price,
-    area,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    title,
-    shortDescription,
-    streetName
-  };
-  //Input type textare to be shown on Additional Data
-  const additionalInputInputs = ADDITIONAL_INPUT_PUBLISH.filter(filterInputs);
-  const additionalInputValues = {
-    longDescription
-  };
-
-  res.render("publishRealEstate", {
-    title: pageTitle,
-    basicBooleanPublishInputs,
-    basicBooleanPublishValues,
-    additionalBooleanPublishInputs,
-    additionalBooleanPublishValues,
-    basicSegmentSelectInputs,
-    basicSegmentSelectValues,
-    additionalSegmentSelectInputs,
-    additionalSegmentSelectValues,
-    basicInputInputs,
-    basicInputValues,
-    additionalInputInputs,
-    additionalInputValues,
-    validate: validate,
-    email,
-    locationLat: locationLat || 0,
-    locationLong: locationLong || 0,
-    editingRealEstate,
-    realEstatePhotosUrls
-  });
-};
-
-const postPublishInputs = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  if (!kiviOriginal || !kiviOriginal.kiviAdId) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const editingRealEstate = req.body.editingRealEstate === "true";
-
-  // console.log("Editing real estate:", editingRealEstate);
-
-  const nextStepPage = editingRealEstate
-    ? req.query.nextStep || "/uspjesnaizmjena"
-    : req.query.nextStep || "/uspjesnaobjava";
-  //Request body
-  //console.log("Body:", req.body);
-
-  const balcony = req.body.balcony === "on";
-  const elevator = req.body.elevator === "on";
-  const newBuilding = req.body.newBuilding === "on";
-  const recentlyAdapted = req.body.recentlyAdapted === "on";
-  const water = req.body.water === "on";
-  const electricity = req.body.electricity === "on";
-  const drainageSystem = req.body.drainageSystem === "on";
-  const registeredInZkBooks = req.body.registeredInZkBooks === "on";
-  const parking = req.body.parking === "on";
-  const garage = req.body.garage === "on";
-  const gas = req.body.gas === "on";
-  const antiTheftDoor = req.body.antiTheftDoor === "on";
-  const airCondition = req.body.airCondition === "on";
-  const phoneConnection = req.body.phoneConnection === "on";
-  const cableTV = req.body.cableTV === "on";
-  const internet = req.body.internet === "on";
-  const basementAttic = req.body.basementAttic === "on";
-  const storeRoom = req.body.storeRoom === "on";
-  const videoSurveillance = req.body.videoSurveillance === "on";
-  const alarm = req.body.alarm === "on";
-  const suitableForStudents = req.body.suitableForStudents === "on";
-  const includingBills = req.body.includingBills === "on";
-  const animalsAllowed = req.body.animalsAllowed === "on";
-  const pool = req.body.pool === "on";
-  const exchange = req.body.exchange === "on";
-  const urbanPlanPermit = req.body.urbanPlanPermit === "on";
-  const buildingPermit = req.body.buildingPermit === "on";
-
-  const furnishingType = req.body.furnishingType;
-  //VALIDACIJA TAKO POTVRDITI DA JE ISPRAVNA VRIJEDNOST
-  /* if (!FURNISHING_TYPE[furnishingType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  } */
-  const accessRoadType = req.body.accessRoadType;
-  /*if (!ACCESS_ROAD_TYPE[accessRoadType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  } */
-  const heatingType = req.body.heatingType;
-  /*if (!HEATING_TYPE[heatingType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  }*/
-
-  const price = parseFloat(req.body.price) || null;
-  const area = parseFloat(req.body.area) || null;
-  const gardenSize = parseFloat(req.body.gardenSize) || null;
-  const numberOfRooms = parseInt(req.body.numberOfRooms) || null;
-  const numberOfFloors = parseInt(req.body.numberOfFloors) || null;
-  const floor = parseInt(req.body.floor) || null;
-  const title = req.body.title || "";
-  const shortDescription = req.body.shortDescription || "";
-  const streetName = req.body.streetName || "";
-  const longDescription = req.body.longDescription || "";
-
-  const locationLat = req.body.lat || null;
-  const locationLong = req.body.lng || null;
-  //Contact email saved in other table
-  const contactEmail = req.body.email || "";
-  //Image urls are stored in new table
-  const imageUrls =
-    req.body.imageUrls.split("|").filter(url => url !== "") || [];
-  //If we are in editing mode we need to "delete" photos that are not longer associated with real estate ad
-  if (editingRealEstate) {
-    await deleteUrlPhotosAfterUpdate(imageUrls);
-  }
-
-  const imageUrlsData = imageUrls.map(url => {
-    return {
-      kiviAdId: kiviOriginal.kiviAdId,
-      photoUrl: url
-    };
-  });
-  const savedImageUrls = await bulkUpsertKiviPhotos(imageUrlsData);
-
-  realEstate.balcony = balcony;
-  realEstate.elevator = elevator;
-  realEstate.newBuilding = newBuilding;
-  realEstate.recentlyAdapted = recentlyAdapted;
-  realEstate.water = water;
-  realEstate.electricity = electricity;
-  realEstate.drainageSystem = drainageSystem;
-  realEstate.registeredInZkBooks = registeredInZkBooks;
-  realEstate.parking = parking;
-  realEstate.garage = garage;
-  realEstate.gas = gas;
-  realEstate.antiTheftDoor = antiTheftDoor;
-  realEstate.airCondition = airCondition;
-  realEstate.phoneConnection = phoneConnection;
-  realEstate.cableTV = cableTV;
-  realEstate.internet = internet;
-  realEstate.basementAttic = basementAttic;
-  realEstate.storeRoom = storeRoom;
-  realEstate.videoSurveillance = videoSurveillance;
-  realEstate.alarm = alarm;
-  realEstate.suitableForStudents = suitableForStudents;
-  realEstate.includingBills = includingBills;
-  realEstate.animalsAllowed = animalsAllowed;
-  realEstate.pool = pool;
-  realEstate.exchange = exchange;
-  realEstate.urbanPlanPermit = urbanPlanPermit;
-  realEstate.buildingPermit = buildingPermit;
-
-  realEstate.furnishingType = furnishingType;
-  realEstate.accessRoadType = accessRoadType;
-  realEstate.heatingType = heatingType;
-
-  realEstate.price = price;
-  realEstate.area = area;
-  realEstate.gardenSize = gardenSize;
-  realEstate.numberOfRooms = numberOfRooms;
-  realEstate.numberOfFloors = numberOfFloors;
-  realEstate.floor = floor;
-  realEstate.title = title;
-  realEstate.shortDescription = shortDescription;
-  realEstate.streetName = streetName;
-
-  realEstate.longDescription = longDescription;
-
-  realEstate.locationLat = locationLat;
-  realEstate.locationLong = locationLong;
-
-  kiviOriginal.email = contactEmail;
-
-  //console.log("realEstate", realEstate);
-
-  await realEstate.save();
-
-  await kiviOriginal.save();
-
-  //Calling function to notify real estate owner that ads is published or edited on Kivi page after 1 sec
-  setTimeout(
-    notifyForNewAdPublish,
-    1000,
-    realEstate,
-    kiviOriginal,
-    editingRealEstate
-  );
-  //Calling function to notify users of new real estate (or edited realestate) after 2 min
-  setTimeout(notifyForNewRealEstates, 1000 * 60 * 2, [realEstate]);
-
-  res.redirect(nextStepPage);
-};
-
-module.exports = {
-  getPublishInputs,
-  postPublishInputs
-};

app/controllers/publishRealEstateTypes.js

@@ -1,105 +0,0 @@
-const { currentKiviRealEstate } = require("../helpers/url");
-const {
-  createRealEstate,
-  findRealEstateByAgencyId
-} = require("../helpers/db/realEstate");
-const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
-
-const { AD_CATEGORY, AD_TYPE, AD_AGENCY } = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
-
-const getPublishTypes = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  const title = "Koju nekretninu nudite?";
-  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
-  const labelAdType = ["Prodaj", "Iznajmi"];
-
-  if (
-    realEstate &&
-    realEstate.adType &&
-    realEstate.adType === AD_TYPE.AD_TYPE_RENT.stringId
-  ) {
-    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
-  }
-  const realEstateTypes = Object.keys(AD_CATEGORY)
-    .map(category => AD_CATEGORY[category])
-    .filter(category => category.title);
-
-  res.render("realEstateType", {
-    selectedAdType,
-    labelAdType,
-    realEstateTypes,
-    title,
-    AD_TYPE
-  });
-};
-
-const postPublishTypes = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  const adType = parseInt(req.body.adType);
-
-  const adTypeStringIds = {
-    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
-    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
-  };
-
-  const adTypeStringId =
-    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
-
-  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
-    category => !!AD_CATEGORY[category].title
-  );
-
-  const selectedRealEstateType = req.body.realEstateType || null;
-  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const nextStepPage = req.query.nextStep || "podacionekretnini";
-
-  let nextStepUrl = "";
-  if (kiviOriginal && kiviOriginal.kiviAdId && realEstate && realEstate.id) {
-    //
-    nextStepUrl = `/${nextStepPage}/${kiviOriginal.kiviAdId}`;
-
-    realEstate.adType = adTypeStringId;
-    realEstate.realEstateType = selectedRealEstateType;
-    //Url override
-    realEstate.url = `${APP_URL}/preglednekretnine/${realEstate.id}`;
-
-    await realEstate.save();
-  } else {
-    try {
-      const newKiviOriginal = await createKiviOriginal({
-        email: ""
-      });
-      const newKiviAdViewUrl = `${APP_URL}/preglednekretnine/${realEstate.id}`;
-
-      const newRealEstate = await createRealEstate({
-        adType: adTypeStringId,
-        realEstateType: selectedRealEstateType,
-        url: newKiviAdViewUrl,
-        originAgencyName: AD_AGENCY.KIVI,
-        agencyObjectId: newKiviOriginal.kiviAdId
-      });
-
-      nextStepUrl = `/${nextStepPage}/${newKiviOriginal.kiviAdId}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  }
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getPublishTypes,
-  postPublishTypes
-};

app/controllers/publishSuccess.js

@@ -1,8 +0,0 @@
-const publishSuccess = async (req, res) => {
-  const title = "Uspjeh!";
-  res.render("publishSuccess", { title });
-};
-
-module.exports = {
-  publishSuccess
-};

app/controllers/realEstateFilters.js

@@ -122,8 +122,6 @@ const getFilters = async (req, res) => {
 };
 
 const postFilters = async (req, res) => {
-  //
-  console.log("postFilters");
   const searchRequest = await currentSearchRequest(req);
 
   if (!searchRequest || !searchRequest.dataValues) {

app/controllers/realEstateTypes.js

@@ -8,7 +8,6 @@ const getRealEstateTypes = async (req, res) => {
 
   const title = "Koju nekretninu tražite?";
   let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
-  const labelAdType = [AD_TYPE.AD_TYPE_SALE.title, AD_TYPE.AD_TYPE_RENT.title];
   if (
     searchRequest &&
     searchRequest.adType &&
@@ -22,7 +21,6 @@ const getRealEstateTypes = async (req, res) => {
 
   res.render("realEstateType", {
     selectedAdType,
-    labelAdType,
     realEstateTypes,
     title,
     AD_TYPE

app/controllers/viewRealEstate.js

@@ -1,202 +0,0 @@
-const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
-const { findPhotosForKiviAd } = require("../helpers/db/kiviOriginalAdsPhotos");
-const { currentKiviRealEstate, currentRealEstate } = require("../helpers/url");
-
-const {
-  BASIC_BOOLEAN_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH,
-  BASIC_INPUT_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH
-} = require("../common/publishEnums");
-
-const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
-
-const getViewRealEstate = async (req, res) => {
-  //Variation if we acces to real estate previews via kiviAdId
-  /*
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  if (!kiviOriginal || !kiviOriginal.kiviAdId) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId); */
-
-  const realEstate = await currentRealEstate(req);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const pageTitle = "Pregled nekretnine";
-
-  const {
-    price,
-    area,
-    adType,
-    agencyObjectId,
-    realEstateType,
-    locationLat,
-    locationLong,
-    accessRoadType,
-    heatingType,
-    balcony,
-    newBuilding,
-    elevator,
-    recentlyAdapted,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit,
-    furnishingType,
-    shortDescription,
-    streetName,
-    title,
-    longDescription
-  } = realEstate;
-  //Categorize all database values by value type - input, boolean or segment selected
-  const allInputValues = {
-    price,
-    area,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    title,
-    shortDescription,
-    streetName,
-    longDescription
-  };
-  const allBooleanValues = {
-    balcony,
-    elevator,
-    newBuilding,
-    recentlyAdapted,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit
-  };
-
-  const allSegmentSelectedValues = {
-    furnishingType,
-    accessRoadType,
-    heatingType
-  };
-
-  //We need titles of fields ex Balkon, Novogradnja
-  const ALL_BOOLEAN_FIELDS = [
-    ...BASIC_BOOLEAN_PUBLISH,
-    ...ADDITIONAL_BOOLEAN_PUBLISH
-  ];
-  const ALL_INPUT_FIELDS = [
-    ...BASIC_INPUT_PUBLISH,
-    ...ADDITIONAL_INPUT_PUBLISH
-  ];
-  const ALL_SEGMENT_FIELDS = [
-    ...BASIC_SEGMENT_PUBLISH,
-    ...ADDITIONAL_SEGMENT_PUBLISH
-  ];
-
-  //On view add page we will show only values that are not - null, or "", or undefined
-  const forShowing = value => {
-    return value !== false && value !== null && value !== "";
-  };
-  //Filter all values to be shown on page or not
-  //For showing on page we also need title ex. "Balkon"
-  const booleanFields = ALL_BOOLEAN_FIELDS.filter(object => {
-    return forShowing(allBooleanValues[object.dbField]);
-  });
-  const inputFields = ALL_INPUT_FIELDS.filter(object => {
-    return forShowing(allInputValues[object.dbField]);
-  });
-  const segmentFields = ALL_SEGMENT_FIELDS.filter(object => {
-    return forShowing(allSegmentSelectedValues[object.dbField]);
-  });
-
-  //Photo urls from Google storage bucket
-  const kiviAdId = agencyObjectId;
-  const urlGooglePrefix =
-    "https://storage.cloud.google.com/marketalarm-photos/";
-  const realEstatePhotosData = await findPhotosForKiviAd(kiviAdId);
-  const realEstatePhotosUrls = realEstatePhotosData.map(row => {
-    return urlGooglePrefix + row.dataValues.photoUrl;
-  });
-
-  const showRealEstateType = AD_CATEGORY[realEstateType].title.toUpperCase();
-  let showAdType = "";
-  switch (adType) {
-    case AD_TYPE.AD_TYPE_SALE.stringId:
-      showAdType = AD_TYPE.AD_TYPE_SALE.title.toUpperCase();
-      break;
-    case AD_TYPE.AD_TYPE_RENT.stringId:
-      showAdType = AD_TYPE.AD_TYPE_RENT.title.toUpperCase();
-      break;
-    default:
-      showAdType = "-";
-      break;
-  }
-
-  res.render("viewRealEstate", {
-    title: pageTitle,
-    booleanFields,
-    inputFields,
-    allInputValues,
-    segmentFields,
-    allSegmentSelectedValues,
-    locationLat,
-    locationLong,
-    showAdType,
-    showRealEstateType,
-    realEstatePhotosUrls
-  });
-};
-
-module.exports = {
-  getViewRealEstate
-};

app/controllers/welcome.js

@@ -1,8 +1,6 @@
 const { createSearchRequest } = require("../helpers/db/searchRequest");
-const { createRealEstate } = require("../helpers/db/realEstate");
-const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
-const { AD_TYPE, AD_CATEGORY, AD_AGENCY } = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
+
+const { AD_TYPE, AD_CATEGORY } = require("../common/enums");
 
 const getWelcome = (req, res) => {
   res.render("welcome", {
@@ -13,55 +11,7 @@ const getWelcome = (req, res) => {
 
 const postWelcome = async (req, res) => {
   const adType = parseInt(req.body.adType);
-  const publishAdType = parseInt(req.body.publishAdType);
 
-  let nextStepUrl = "";
-
-  if (adType) {
-    const adTypeStringId = getAdTypeString(adType);
-    try {
-      const newSearchRequest = await createSearchRequest({
-        adType: adTypeStringId,
-        realEstateType: AD_CATEGORY.FLAT.id
-      });
-
-      nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  } else if (publishAdType) {
-    const adTypeStringId = getAdTypeString(publishAdType);
-
-    try {
-      //First we create new Kivi Ad Original object in db then new Real Estate
-      //Problem with id-s
-      const newKiviOriginal = await createKiviOriginal({
-        email: ""
-      });
-      //Temporary url because we have cyclic id call - need to override for safety measures
-      const newKiviAdViewUrl = `${APP_URL}/preglednekretnine/${newKiviOriginal.kiviAdId}`;
-
-      const newRealEstate = await createRealEstate({
-        adType: adTypeStringId,
-        realEstateType: AD_CATEGORY.FLAT.id,
-        url: newKiviAdViewUrl,
-        originAgencyName: AD_AGENCY.KIVI,
-        agencyObjectId: newKiviOriginal.kiviAdId
-      });
-
-      nextStepUrl = `/objavinekretninu/${newKiviOriginal.kiviAdId}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  }
-
-  res.redirect(nextStepUrl);
-};
-
-//--- Helper function
-const getAdTypeString = adType => {
   const adTypeStringIds = {
     [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
     [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
@@ -70,7 +20,20 @@ const getAdTypeString = adType => {
   const adTypeStringId =
     adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
 
-  return adTypeStringId;
+  let nextStepUrl = "";
+  try {
+    const newSearchRequest = await createSearchRequest({
+      adType: adTypeStringId,
+      realEstateType: AD_CATEGORY.FLAT.id
+    });
+
+    nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
+  } catch (error) {
+    console.log(error);
+    nextStepUrl = `/`;
+  }
+
+  res.redirect(nextStepUrl);
 };
 
 module.exports = {

app/crawler/crawl.js

@@ -10,6 +10,7 @@ const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
 const SaljicCrawler = require("./specificCrawlers/saljic");
+const { logDebug } = require("../helpers/log");
 
 const {
   OLX_CONFIG,
@@ -75,7 +76,9 @@ async function crawlAll() {
 
   for (const crawler of crawlers) {
     try {
+      logDebug('Starting crawler: ', crawler);
       const newRealEstatesFromSingleCrawler = await crawler.crawl();
+      logDebug('Crawler done: ', crawler);
       if (Array.isArray(newRealEstatesFromSingleCrawler)) {
         newRealEstates.push(...newRealEstatesFromSingleCrawler);
       }

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -159,7 +159,7 @@ class AktidoCrawler {
     }
 
     try {
-      const res = await fetch(url);
+      const res = await fetch(url, {}, false);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -202,6 +202,10 @@ class AktidoCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (

app/crawler/specificCrawlers/olx.js

@@ -1,6 +1,7 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
+const { logDebug } = require("../../helpers/log");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -44,6 +45,16 @@ const OLX_ENUMS = {
 
 const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
 
+const chunk = (array, size = 10) => {
+  let i, j ,temparray;
+  const result = []
+  for (i=0,j=array.length; i<j; i+=size) {
+    temparray = array.slice(i,i+size);
+    result.push(temparray);
+  }
+  return result;
+}
+
 class OlxCrawler {
   constructor(
     savers = [],
@@ -52,7 +63,7 @@ class OlxCrawler {
     maxPages = 1000,
     maxResultsPerPage = 100,
     ignoredUsernames = [],
-    delayBetweenPages = 1000
+    delayBetweenPages = 500
   ) {
     this.savers = savers;
     this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
@@ -65,6 +76,7 @@ class OlxCrawler {
   }
 
   async crawl() {
+    logDebug("Starting OLX crawl");
     const crawlAdCategories = this.crawlerAdCategories;
 
     const newRealEstates = [];
@@ -88,14 +100,32 @@ class OlxCrawler {
         const entries = singlePageResults.entries();
 
         for (const [index, { value: singlePageResult }] of entries) {
+          if (PRINT_CRAWLER_DEBUG) {
+            console.log("================================");
+            console.log("Category Indexer index : ", index);
+          }
+
           if (singlePageResult) {
+            console.log("\tTotal entries          : ", singlePageResult.length)
             const saveResults = await this.saveCrawledResults(singlePageResult);
             const { newRecords, existingRecords } = saveResults;
 
+            if (PRINT_CRAWLER_DEBUG) {
+              console.log("--------------------------");
+              console.log("\tNew record URLs [", newRecords.length, "] :");
+
+              for(const newRecord of newRecords) {
+                console.log("\t\t",newRecord.url);
+              }
+
+              console.log("\t-------------------------");
+              console.log("\tExisting record URLs [", existingRecords.length, "] :");
+            }
+
             newRealEstates.push(...newRecords);
 
             for (const existingRecord of existingRecords) {
-              const { publishedDate, renewedDate } = existingRecord;
+              const { publishedDate, renewedDate, url } = existingRecord;
 
               const publishedDateMoment = moment.utc(publishedDate);
               const renewedDateMoment = moment.utc(renewedDate);
@@ -105,13 +135,25 @@ class OlxCrawler {
                 "minute"
               );
 
+              if (PRINT_CRAWLER_DEBUG) {
+                console.log("\t\t", url);
+                console.log("\t\t\tPublished date         : ", publishedDate);
+                console.log("\t\t\tRenewed date           : ", renewedDate);
+                console.log("\t\t\tIs same (up to minute) : ", stopCrawlingThisCategory);
+              }
+
               if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
                 generatorsToRemove[index] = true;
-                // console.log("\tGenerator ", index + 1, "has no more new ads");
+                if (PRINT_CRAWLER_DEBUG) {
+                  console.log("\t\t\tStopping this category indexer");
+                }
                 break;
               }
             }
           } else {
+            if (PRINT_CRAWLER_DEBUG) {
+              console.log("\tNo more entries in this category, stopping!");
+            }
             //Generator returned undefined, remove this generator from array
             generatorsToRemove[index] = true;
             // console.log("Generator ", index + 1, "has no more pages");
@@ -136,31 +178,36 @@ class OlxCrawler {
   }
 
   async *categoryIndexer(adCategory) {
-    let pageToIndex = 1;
+    try {
+      let pageToIndex = 1;
 
-    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
-    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
-    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
-      while (true) {
-        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
-        const singlePageResults = await this.indexSinglePage(
-          urlPageToCrawl,
-          this.maxResultsPerPage
-        );
+      const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
+      const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
+      if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+        while (true) {
+          const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
+          const singlePageResults = await this.indexSinglePage(
+            urlPageToCrawl,
+            this.maxResultsPerPage
+          );
 
-        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
-          yield singlePageResults;
-        } else {
-          return undefined;
-        }
-
-        ++pageToIndex;
-        if (pageToIndex === this.maxPages) {
-          return undefined;
+          await this.sleep(this.delayBetweenPages);
+          if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+            yield singlePageResults;
+          } else {
+            return undefined;
+          }
+
+          ++pageToIndex;
+          if (pageToIndex === this.maxPages) {
+            return undefined;
+          }
         }
+      } else {
+        return undefined;
       }
-    } else {
-      return undefined;
+    } catch (e) {
+      console.log('Error inside generator: ', e);
     }
   }
 
@@ -170,8 +217,10 @@ class OlxCrawler {
     }
 
     try {
-      const res = await fetch(url);
+      const res = await fetch(url, {}, false);
+      logDebug("Got category results for: ", url);
       const body = await res.text();
+      logDebug("Got category results text for: ", url);
       const $ = cheerio.load(body);
       let hrefs = [];
 
@@ -192,26 +241,44 @@ class OlxCrawler {
 
       const asyncScraping = [];
       for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefs[i]));
+        asyncScraping.push(hrefs[i]);
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
-      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      const allChunks = chunk(asyncScraping, 2);
+      const dataResults = []
+      for (let i = 0; i < allChunks.length; i++) {
+        const singleChunk = allChunks[i];
+        const promises = singleChunk.map(c => this.scrapeAd(c))
+        const chunkResults = await Promise.all(promises);
+        await this.sleep(this.delayBetweenPages);
+        dataResults.push(...chunkResults);
+        logDebug("Chunk results len:", chunkResults.length);
+      }
+
+
+      const filteredScrapedData = dataResults.filter(adData => !!adData);
+      logDebug("Filtered scraped data length: ", filteredScrapedData.length);
+
       return filteredScrapedData;
     } catch (e) {
-      console.error("Exception caught:" + e);
+      console.error("Exception caught, index single page: " + e);
       return [];
     }
   }
 
   async scrapeAd(url) {
-    // console.log("Scraping : ", url);
+    logDebug("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
       let status = AD_STATUS.STATUS_NORMAL;
 
+      if (body.indexOf('<html') === -1) {
+        console.error("This is the body: ", body);
+        throw { message: 'Failed to fetch page !' }
+      }
+
       const propertySelectors = {
         username:
           "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
@@ -238,37 +305,25 @@ class OlxCrawler {
 
       //====== PRICE DETECTION AND EXTRACTION =====
       let price = null;
-      const normalPriceValue = $("#pc > p:nth-child(2)").text();
-      const urgentPriceValue = $(
-        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
-      )
-        .text()
-        .trim();
 
-      if (normalPriceValue && normalPriceValue.length > 0) {
-        price = normalPriceValue;
-        if (
-          $("#pc > p.n")
-            .text()
-            .indexOf("Hitna") !== -1
-        ) {
-          status = AD_STATUS.STATUS_URGENT;
-        } else {
-          status = AD_STATUS.STATUS_NORMAL;
-        }
-      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
-        const priceValues = urgentPriceValue.split("KM");
-        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
-        if (priceValues.length > 1) {
-          price = priceValues[1].trim();
-          status = AD_STATUS.STATUS_DISCOUNTED;
-        } else {
-          throw { message: "Can't find urgent price" };
-        }
-      } else {
-        throw {
-          message: "Can't find price (it is not normal nor urgent price ?)"
-        };
+      const priceHeader = $("#pc > p.n").text().trim();
+      const priceValue = $("#pc > p:nth-child(2)").text().trim();
+      price = priceValue;
+
+      if (priceHeader.indexOf('Hitn') !== -1) {
+        // Urgent price
+        status = AD_STATUS.STATUS_URGENT;
+      }
+
+      const discountPriceTag = $("#artikal_glavni_div > div.artikal_lijevo > p:nth-child(4)").text().trim();
+      if (discountPriceTag.indexOf('Akcij') !== -1) {
+        status = AD_STATUS.STATUS_DISCOUNTED;
+        const discountPriceValues = $("#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p").text().trim();
+        // discountPriceValues contain string like "10.000 KM 7.500 KM"
+        // First price is regular, second is currently active (discounted) price
+        const bothPrices = discountPriceValues.split('KM');
+        // Now, currently active price is second element of bothPrices array
+        price = bothPrices[1] ? bothPrices[1].trim() : null;
       }
 
       //====== OTHER AD INFORMATION ===============
@@ -278,7 +333,7 @@ class OlxCrawler {
 
       let otherInformationDivId;
       //We need to locate DIV ID where other information are stored
-      for (let possibleId = 10; possibleId <= 20; possibleId++) {
+      for (let possibleId = 1; possibleId <= 30; possibleId++) {
         const adTypeFieldTitle = $(
           `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
         )
@@ -650,10 +705,12 @@ class OlxCrawler {
         distanceToRiver,
         numberOfViewsAgency
       };
+      //
+      //console.log("Scraped data:", data);
 
       return data;
     } catch (e) {
-      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+      console.error("Exception caught scrapeAd : " + e.message, "\r\nURL:", url);
     }
     return null;
   }
@@ -768,6 +825,9 @@ class OlxCrawler {
     if (!priceText) {
       return NaN;
     }
+    if (priceText === "Po dogovoru") {
+      return null;
+    }
     const formattedPriceText = priceText.replace(".", "").replace(",", ".");
     return parseFloat(formattedPriceText);
   }

app/crawler/specificCrawlers/prostor.js

@@ -1,9 +1,10 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
+const nodeFetch = require("node-fetch");
 
 const {
   AD_TYPE,
@@ -62,13 +63,19 @@ class ProstorCrawler {
 
   async crawl() {
     const crawlAdCategories = this.crawlerAdCategories;
+    const crawlAdTypes = this.crawlerAdTypes;
+    if (!crawlAdCategories || !crawlAdTypes) {
+      return []
+    }
+
+    const newRealEstates = [];
     //We need session cookie to use login privileges
     const prostorCookie = await this.getCookies();
-    //New tag to check if crawler loged in
+    //New tag to check if crawler logged in
     const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
-    const newRealEstates = [];
+
     //Crawl only if login was successful
-    if (crawlAdCategories && login) {
+    if (login) {
       const indexGenerators = [];
       for (const adCategory of crawlAdCategories) {
         indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
@@ -134,6 +141,11 @@ class ProstorCrawler {
         prostorCookie
       );
 
+      if (!Array.isArray(listOfAllRealEstates)){
+        console.log('[PROSTOR] Could not find real estate JSON data, check selector !');
+        return undefined;
+      }
+
       let elementToStartIndexFrom = 0;
       while (true) {
         const realEstatesForSinglePage = listOfAllRealEstates.slice(
@@ -191,24 +203,22 @@ class ProstorCrawler {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
 
     //Status information is given already in realestate list
-    //For VIP Ads status ='' canot be used, but no VIP ads are crawled
-    //We will make "fake" vip ad for RE that have size=55
-    //It is weird because yesterday it said 'VIP ponuda' ???
-    const adStatus =
-      size === "55"
-        ? ProstorCrawler.getStatusId("VIP ponuda")
-        : ProstorCrawler.getStatusId(status);
+    const adStatus = ProstorCrawler.getStatusId(status);
 
     const url = `https://prostor.ba${link}`;
 
     // console.log("[PROSTOR] Scraping : ", url);
     try {
-      const adPageSource = await fetch(url, {
+      const adPageSource = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       // link contains part of the URL in the format of : /prodaja/stan/stup/9556
       // general form is : /actionType/realEstateType/location/realEstateID
       // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
@@ -433,14 +443,14 @@ class ProstorCrawler {
     }
 
     try {
-      const res = await fetch(url, {
+      const res = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await res.text();
       const $ = cheerio.load(body);
 
       const scriptElement = $(
-        "body > div > div.container-fluid > script:nth-child(7)"
+          "body > div.content > div.container-fluid > script:nth-child(6)"
       );
 
       if (
@@ -597,7 +607,7 @@ class ProstorCrawler {
     formData.append("email", PROSTOR_LOGIN.EMAIL);
     formData.append("password", PROSTOR_LOGIN.PASSWORD);
 
-    return fetch("https://prostor.ba/moj-prostor/prijava", {
+    return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
       method: "POST",
       body: formData,
       headers: { Cookie: prostorCookie }
@@ -624,9 +634,12 @@ class ProstorCrawler {
       });
   }
   async getCookies() {
-    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
-      headers: { Cookie: "" }
-    });
+    const getResponse = await nodeFetch(
+      "https://prostor.ba/moj-prostor/prijava",
+      {
+        headers: { Cookie: "" }
+      }
+    );
     const raw = getResponse.headers.raw()["set-cookie"];
     const cookie = raw
       .map(datastring => {

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -159,7 +159,7 @@ class RentalCrawler {
     }
 
     try {
-      const res = await fetch(url);
+      const res = await fetch(url, {} , false);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -202,6 +202,10 @@ class RentalCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (
@@ -399,7 +403,9 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
           };
         }
 
@@ -410,7 +416,9 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -1,8 +1,10 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
+const { getUrlParams } = require("../../helpers/url");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
+const PromisePool = require('@supercharge/promise-pool');
 
 const {
   AD_TYPE,
@@ -46,12 +48,13 @@ class SaljicCrawler {
     maxPages = 5000,
     maxResultsPerPage = 5000,
     ignoredUsernames = [],
-    delayBetweenPages = 1000
+    delayBetweenPages = 500
   ) {
     this.savers = savers;
     this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
     this.crawlerAdTypes = crawlerAdTypes;
     this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages
     this.maxResultsPerPage = maxResultsPerPage;
     this.delayBetweenPages = delayBetweenPages;
   }
@@ -157,7 +160,7 @@ class SaljicCrawler {
     }
 
     try {
-      const res = await fetch(url);
+      const res = await fetch(url, {}, false);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -205,11 +208,23 @@ class SaljicCrawler {
 
       const asyncScraping = [];
       for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+        asyncScraping.push([hrefsAbs[i], adTypes[i]]);
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
-      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      
+
+      const dataResults = []
+      const { scrapedData, errors } = await PromisePool
+        .withConcurrency(2)
+        .for(asyncScraping)
+        .process(async data => {
+          const result = await this.scrapeAd(...data)
+          await this.sleep(this.delayBetweenPages);
+          dataResults.push(result)
+          return result; //TODO: this does not work, scrapedData is null, dataResults works
+        })
+
+      const filteredScrapedData = dataResults.filter(adData => !!adData);
       return filteredScrapedData;
     } catch (e) {
       console.error("[SALJIC] Exception caught:" + e);
@@ -224,10 +239,20 @@ class SaljicCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      if (body.indexOf('<html') === -1) {
+        throw { message: 'Failed to fetch page !' }
+      }
+
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
+      const agencyObjectId = url
+        ? parseInt(url.substring(46, url.length))
+        : null;
+
+      if (!agencyObjectId) {
+        throw { message : 'No agency object ID - URL changed?'}
+      }
 
       //Extracting main properties
       const propertySelectors = {
@@ -237,11 +262,10 @@ class SaljicCrawler {
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
         streetName:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
-
         descriptions:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
         latAndLong:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
+          "iframe"
       };
       const title = $(propertySelectors.title)
         .text()
@@ -272,14 +296,26 @@ class SaljicCrawler {
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
+      let latText;
+      let longText;
+      if (latAndLongSrc){
+        const mapParams = getUrlParams(latAndLongSrc);
+        if (mapParams) {
+          if (mapParams['marker']){
+            const marker = mapParams['marker'].split(',');
+            latText = marker[0] ? marker[0] : undefined;
+            longText = marker[1] ? marker[1] : undefined;
+          }else{
+            if (mapParams['mlat']) {
+              latText = mapParams['mlat'];
+            }
+            if (mapParams['mlon']) {
+              longText = mapParams['mlon'];
+            }
+          }
+        }
+      }
+
       const locationLat = parseFloat(latText) || null;
       const locationLong = parseFloat(longText) || null;
 
@@ -328,11 +364,10 @@ class SaljicCrawler {
       let numberOfViewsKivi = null;
       let streetNumber = 0;
       let adStatus = status;
-      let shortDescription = descriptions.substring(
-        0,
-        descriptions.indexOf(".")
-      );
-      let longDescription = descriptions;
+      let shortDescription = descriptions
+        ? descriptions.substring(0, descriptions.indexOf("."))
+        : "";
+      let longDescription = descriptions || "";
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -343,10 +378,14 @@ class SaljicCrawler {
           .replace(/[\n\r\t]/gm, "")
           .trim();
 
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldTitle = mainField
+          ? mainField.substring(0, mainField.indexOf("  "))
+          : "";
         const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
+          ? mainField
+              .substring(mainField.indexOf("  "), mainField.length)
+              .trim()
+          : "";
 
         switch (mainFieldTitle) {
           case "Površina":
@@ -363,7 +402,7 @@ class SaljicCrawler {
             numberOfRooms = parseInt(mainFieldValue);
             break;
           case "Broj spratova":
-            numberOfFloors = parseInt(mainFieldValue);
+            numberOfFloors = this.parseNumberOfFloors(mainFieldValue);
             break;
           case "Sprat":
             floor = parseInt(mainFieldValue);
@@ -409,6 +448,9 @@ class SaljicCrawler {
             )
             .trim();
           realEstateType = this.getAdCategoryId(categoryTmp);
+          if (!realEstateType) {
+            throw { message: 'No real estate type - page body not loaded correctly or page changed?' }
+          }
         } else {
           switch (additionalField) {
             case "Internet":
@@ -565,7 +607,7 @@ class SaljicCrawler {
 
       return data;
     } catch (e) {
-      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+      console.error("[SALJIC] Exception caught: " + e.message, "\r\nURL:", url);
     }
     return null;
   }
@@ -610,6 +652,21 @@ class SaljicCrawler {
     }
   }
 
+  parseNumberOfFloors(numberOfFloorsText) {
+    const tryNumericalValue = parseInt(numberOfFloorsText);
+    if (!isNaN(tryNumericalValue)){
+      return tryNumericalValue;
+    }
+
+    // Guess number of floors based on number of + sign concatenations
+    // e.g. P+S+Pt -> 3 floors
+    if (typeof numberOfFloorsText === 'string' && numberOfFloorsText.indexOf('+') > 0) {
+      return numberOfFloorsText.split('+').length + 1
+    }
+
+    return null
+  }
+
   async sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }

app/helpers/db/kiviOriginal.js

@@ -1,20 +0,0 @@
-"use strict";
-const db = require("../../models/index");
-const sequelize = require("sequelize");
-
-const createKiviOriginal = async (kiviAdFields = {}) => {
-  return await db.KiviOriginal.create(kiviAdFields);
-};
-
-const getKiviOriginalById = async id => {
-  try {
-    return db.KiviOriginal.findByPk(id);
-  } catch (error) {
-    console.log("kiviOriginal.js", error);
-    return null;
-  }
-};
-module.exports = {
-  createKiviOriginal,
-  getKiviOriginalById
-};

app/helpers/db/kiviOriginalAdsPhotos.js

@@ -1,51 +0,0 @@
-"use strict";
-const db = require("../../models/index");
-
-const sequelize = require("sequelize");
-const Op = sequelize.Op;
-
-const bulkUpsertKiviPhotos = async kiviPhotosData => {
-  try {
-    return await db.KiviOriginalAdsPhotos.bulkCreate(kiviPhotosData, {
-      ignoreDuplicates: true
-    });
-  } catch (e) {
-    console.log("Error bulk upserting kiviOriginalAdsPhotos : ", e);
-  }
-};
-
-const findPhotosForKiviAd = async id => {
-  try {
-    return db.KiviOriginalAdsPhotos.findAll({
-      where: {
-        kiviAdId: id
-      }
-    });
-  } catch (error) {
-    console.log("kiviOriginalAdsPhotos.js", error);
-    return null;
-  }
-};
-
-const deleteUrlPhotosAfterUpdate = async photoUrlsToKeep => {
-  //We delete all urls that are not in "newly changed - edited" photo urls array
-  const deleteQuery = {
-    photoUrl: {
-      [Op.notIn]: photoUrlsToKeep
-    }
-  };
-  try {
-    return db.KiviOriginalAdsPhotos.destroy({
-      where: deleteQuery
-    });
-  } catch (error) {
-    console.log("kiviOriginalAdsPhotos.js", error);
-    return null;
-  }
-};
-
-module.exports = {
-  bulkUpsertKiviPhotos,
-  findPhotosForKiviAd,
-  deleteUrlPhotosAfterUpdate
-};

app/helpers/db/realEstate.js

@@ -77,28 +77,9 @@ const bulkUpsertRealEstates = async realEstateData => {
 };
 
 const getRealEstateById = async id => {
-  try {
-    return db.RealEstate.findByPk(id);
-  } catch (error) {
-    console.log("realEstate.js", error);
-    return null;
-  }
+  return db.RealEstate.findByPk(id);
 };
 
-const createRealEstate = async (realEstateFields = {}) => {
-  return await db.RealEstate.create(realEstateFields);
-};
-
-const findRealEstateByAgencyId = async kiviId => {
-  try {
-    return db.RealEstate.findOne({
-      where: { agencyObjectId: kiviId }
-    });
-  } catch (error) {
-    console.log("realEstate.js", error);
-    return null;
-  }
-};
 const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const {
     priceMin,
@@ -350,18 +331,15 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-  //Query only for real estated that are not deleted
-  query.deleted = {
-    [Op.eq]: false
-  };
-  queryIncludeIncomplete.deleted = {
-    [Op.eq]: false
-  };
 
+  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where: includeIncompleteAds ? queryIncludeIncomplete : query,
+    where:
+      includeIncompleteAds || includeIncompleteAds == null
+        ? queryIncludeIncomplete
+        : query,
     limit: maxResults,
     order
   });
@@ -370,7 +348,5 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
 module.exports = {
   bulkUpsertRealEstates,
   getRealEstateById,
-  createRealEstate,
-  findRealEstatesForSearchRequest,
-  findRealEstateByAgencyId
+  findRealEstatesForSearchRequest
 };

app/helpers/db/searchRequest.js

@@ -3,6 +3,7 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -16,6 +17,22 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
+const findAllRequestsForCheckUp = async () => {
+  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
+  const checkupDate = new Date();
+  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
+
+  const dateQuery = {
+    notifiedAt: {
+      [Op.lte]: checkupDate
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: dateQuery
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -157,7 +174,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
+      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -176,7 +193,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -226,7 +246,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -275,7 +298,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -313,7 +339,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -347,7 +376,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -381,7 +413,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -423,10 +458,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query
+  //Tag to check if incomplete ads are accepted in query which is default
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.eq]: true
+      [Op.or]: {
+        [Op.eq]: true,
+        [Op.is]: null
+      }
     };
   }
 
@@ -438,5 +476,6 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate
+  findSearchRequestsForRealEstate,
+  findAllRequestsForCheckUp
 };

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -43,42 +42,6 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
-const findAllRequestsForCheckUp = async () => {
-  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate CHECK_UP_DAYS days from now
-  const date = new Date();
-  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
-  date.setDate(checkUpDate);
-  const dateQuery = {
-    createdAt: {
-      [Op.gte]: date
-    }
-  };
-
-  const excludedMatches = await db.SearchRequestMatch.findAll({
-    attributes: ["searchRequestId"],
-    where: dateQuery,
-    order: [["searchRequestId", "ASC"]]
-  });
-
-  const excludedRequestsAll = excludedMatches.map(match => {
-    return match.dataValues.searchRequestId;
-  });
-  //Removing duplicate search request id-s for optimization
-  const excludedRequests = [...new Set(excludedRequestsAll)];
-
-  const query = {
-    subscribed: true,
-    id: {
-      [Op.notIn]: excludedRequests
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: query
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -89,6 +52,5 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches,
-  findAllRequestsForCheckUp
+  findNotNotifiedMatches
 };

app/helpers/emailContentGenerator.js

@@ -9,6 +9,7 @@ const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
 //Tag to recognize staging from development
 const stagingTag = STAGING ? "[STAGING] " : "";
+const wordOfMouthRequest = `Molimo vas <strong>recite svojim prijateljima</strong> za Kivi - što više korisnika budemo imali, moći ćemo više agencija uključiti i više nekretnina imati u bazi. Hvala!`
 
 const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
   return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>
@@ -69,6 +70,9 @@ const generateNotificationEmail = (
   ${moreRealEstates}
   </div>
   <br/>
+  ${wordOfMouthRequest}
+  <br/>
+  <br/>
   ${emailFooter}`;
 };
 
@@ -132,6 +136,10 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
   </div>
   ${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
   <br/>
+  <br/>
+  ${wordOfMouthRequest}
+  <br/>
+  <br/>
   ${emailFooter}`;
 };
 
@@ -202,52 +210,9 @@ const generateCheckUpEmail = searchRequest => {
   ${emailFooter}`;
 };
 
-const generateNewAdPublishEmail = (
-  realEstate,
-  kiviOriginal,
-  editingRealEstate,
-  numberOfMatchingRequests
-) => {
-  let countingPrefix;
-  if (
-    numberOfMatchingRequests === 2 ||
-    numberOfMatchingRequests === 3 ||
-    numberOfMatchingRequests === 4
-  ) {
-    countingPrefix = "postoje";
-  } else {
-    countingPrefix = "postoji";
-  }
-  let countingSufix;
-  if (numberOfMatchingRequests % 10 === 1 && numberOfMatchingRequests !== 11) {
-    countingSufix = "zahtjev";
-  } else {
-    countingSufix = "zahtjeva";
-  }
-
-  const successIntro = editingRealEstate
-    ? `<div>Uspješno ste izmijenili oglas za Vašu nekretninu na <strong>Kivi.ba.</strong><div>`
-    : `<div>Uspješno ste objavili oglas za Vašu nekretninu na <strong>Kivi.ba.</strong><div>`;
-
-  return `<h3>${stagingTag}Zdravo</h3>
-  ${successIntro}
-  <br/>
-  <div>U Kivi bazi trenutno ${countingPrefix} ${numberOfMatchingRequests} ${countingSufix} za nekretninom kao sto je Vaša.</div>
-  <br />
-  <div>Pregledajte Vaš oglas na sljedećem linku: <a href="${realEstate.url}" rel="noreferrer">${realEstate.title}</a><div>
-  <br/>
-  <div>Ako želite izmijeniti detalje oglasa, <a href="${APP_URL}/podacionekretnini/${kiviOriginal.kiviAdId}">izmjenite ovdje.</a></div>
-  <div>Ako želite izbrisati Vaš oglas iz Kivi baze, <a href="${APP_URL}/obrisioglas/${kiviOriginal.kiviAdId}">izbrišite ovdje.</a></div>
-  <br/> 
-  <div>Hvala na ukazanom povjerenju!</div>
-  <br/>
-  <strong>Vaš,<br/>Kivi tim</strong>`;
-};
-
 module.exports = {
   generateNotificationEmail,
   generateNewSearchRequestEmail,
   generateEmailSubject,
-  generateCheckUpEmail,
-  generateNewAdPublishEmail
+  generateCheckUpEmail
 };

app/helpers/fetchWrapper.js

@@ -0,0 +1,58 @@
+const nodeFetch = require("node-fetch");
+const AbortController = require('abort-controller');
+const FetchCache = require('@sozialhelden/fetch-cache').default;
+
+console.log("Fc ", FetchCache)
+
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL,
+  NODE_FETCH_TIMEOUT_MS
+} = require("../config/appConfig");
+
+const timeout = (ms) => {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+const fetchCache = new FetchCache({
+  fetch: nodeFetch,
+  cacheOptions: {
+    // Don't save more than 100 responses in the cache. Allows infinite responses by default
+    maximalItemCount: 10000,
+    // When should the cache evict responses when its full?
+    evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
+    defaultTTL: 6 * 60 * 60 * 1000  // 6 hours
+    // ...see https://github.com/sozialhelden/hamster-cache for all possible options
+  },
+});
+
+
+const fetch = async (url, options = {}, useCache = true) => {
+  const controller = new AbortController();
+
+  const newOptions = Object.assign({}, options);
+  if (!newOptions["headers"]) {
+    newOptions["headers"] = {};
+  }
+
+  newOptions.signal = controller.signal;
+
+ // newOptions["headers"]["User-Agent"] = USER_AGENT;
+
+  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
+  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
+    urlToFetchThroughAPI = url; 
+  }
+
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
+    : url;
+  const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);
+
+  const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
+  return result;
+};
+
+module.exports = fetch;

app/helpers/log.js

@@ -0,0 +1,13 @@
+const {
+  PRINT_CRAWLER_DEBUG
+} = require("../config/appConfig");
+
+const logDebug = (...args) => {
+  if (PRINT_CRAWLER_DEBUG) {
+    console.log(...args);
+  }
+}
+
+module.exports = {
+  logDebug
+};

app/helpers/url.js

@@ -1,7 +1,4 @@
 const { getSearchRequest } = require("./db/searchRequest");
-const { getRealEstateById } = require("./db/realEstate");
-const { getKiviOriginalById } = require("./db/kiviOriginal");
-const validator = require("validator");
 
 const currentSearchRequest = async req => {
   const searchRequestId =
@@ -11,22 +8,25 @@ const currentSearchRequest = async req => {
   return await getSearchRequest(searchRequestId);
 };
 
-const currentRealEstate = async req => {
-  const realEstateId = req && req.params ? req.params["realEstateId"] : null;
-  if (!realEstateId) return null;
-
-  return await getRealEstateById(parseInt(realEstateId));
-};
-const currentKiviRealEstate = async req => {
-  const kiviRealEstateId =
-    req && req.params ? req.params["kiviRealEstateId"] : null;
-  if (!kiviRealEstateId || !validator.isUUID(kiviRealEstateId)) return null;
-
-  return await getKiviOriginalById(kiviRealEstateId);
+const getUrlParams = function (url) {
+  if (typeof url === 'string' && url.length > 0){
+    const params = {};
+    const questionMarkIndex = url.indexOf('?');
+    if (questionMarkIndex === -1) {
+      return undefined;
+    }
+    const query = url.substring(questionMarkIndex+1);
+    const vars = query.split('&');
+    for (let i = 0; i < vars.length; i++) {
+      const pair = vars[i].split('=');
+      params[pair[0]] = decodeURIComponent(pair[1]);
+    }
+    return params;
+  }
+  return undefined;
 };
 
 module.exports = {
   currentSearchRequest,
-  currentRealEstate,
-  currentKiviRealEstate
+  getUrlParams
 };

app/migrations/20200203114630-add-kiviOriginal-table.js

@@ -1,28 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    const tableFields = {
-      kiviAdId: {
-        type: Sequelize.UUID,
-        defaultValue: Sequelize.UUIDV4,
-        allowNull: false,
-        primaryKey: true
-      },
-      email: Sequelize.TEXT,
-      createdAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      },
-      updatedAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      }
-    };
-    return queryInterface.createTable("KiviOriginal", tableFields);
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable("KiviOriginal", {});
-  }
-};

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
+      type: Sequelize.DATE,
+      defaultValue: new Date()
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
+  }
+};

app/migrations/20200312210336-add-table-kivi-photos.js

@@ -1,39 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    const tableFields = {
-      id: {
-        type: Sequelize.BIGINT,
-        autoIncrement: true,
-        primaryKey: true,
-        allowNull: false
-      },
-      kiviAdId: {
-        type: Sequelize.UUID,
-        allowNull: false,
-        references: {
-          model: "KiviOriginal",
-          key: "kiviAdId"
-        }
-      },
-      photoUrl: {
-        type: Sequelize.TEXT,
-        allowNull: false
-      },
-      createdAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      },
-      updatedAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      }
-    };
-    return queryInterface.createTable("KiviOriginalAdsPhotos", tableFields);
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable("KiviOriginalAdsPhotos", {});
-  }
-};

app/migrations/20200319093739-add-columns-deleted-to-realestates.js

@@ -1,22 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return Promise.all([
-      queryInterface.addColumn("RealEstates", "deleted", {
-        type: Sequelize.BOOLEAN,
-        defaultValue: false
-      }),
-      queryInterface.addColumn("RealEstates", "deletedAt", {
-        type: Sequelize.DATE
-      })
-    ]);
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return Promise.all([
-      queryInterface.removeColumn("RealEstates", "deleted"),
-      queryInterface.removeColumn("RealEstates", "deletedAt")
-    ]);
-  }
-};

app/migrations/20200325120224-add-constraint-kiviOriginalAdsPhotos.js

@@ -1,18 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) =>
-    queryInterface.addConstraint(
-      "KiviOriginalAdsPhotos",
-      ["kiviAdId", "photoUrl"],
-      {
-        type: "unique",
-        name: "uniqueKiviAdIdPhoto"
-      }
-    ),
-  down: queryInterface =>
-    queryInterface.removeConstraint(
-      "KiviOriginalAdsPhotos",
-      "uniqueKiviAdIdPhoto"
-    )
-};

app/models/index.js

@@ -16,7 +16,7 @@ config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;
 
 let sequelize;
 if (config.use_env_variable) {
-  sequelize = new Sequelize(process.env[config.use_env_variable], config);
+  sequelize = new Sequelize(process.env[config.use_env_variable] + "?ssl=true", config);
 } else {
   sequelize = new Sequelize(
     config.database,

app/models/kiviOriginal.js

@@ -1,21 +0,0 @@
-"use strict";
-
-module.exports = (sequalize, DataTypes) => {
-  const KiviOriginal = sequalize.define(
-    "KiviOriginal",
-    {
-      kiviAdId: {
-        type: DataTypes.UUID,
-        defaultValue: DataTypes.UUIDV4,
-        allowNull: false,
-        primaryKey: true
-      },
-      email: DataTypes.TEXT
-    },
-    {
-      freezeTableName: true
-    }
-  );
-
-  return KiviOriginal;
-};

app/models/kiviOriginalAdsPhotos.js

@@ -1,43 +0,0 @@
-"use strict";
-
-module.exports = (sequalize, DataTypes) => {
-  const KiviOriginalAdsPhotos = sequalize.define(
-    "KiviOriginalAdsPhotos",
-    {
-      id: {
-        type: DataTypes.BIGINT,
-        autoIncrement: true,
-        primaryKey: true,
-        allowNull: false
-      },
-      kiviAdId: {
-        type: DataTypes.UUID,
-        allowNull: false,
-        unique: "uniqueKiviAdIdPhoto",
-        references: {
-          model: "KiviOriginal",
-          key: "kiviAdId"
-        }
-      },
-      photoUrl: {
-        type: DataTypes.TEXT,
-        allowNull: false,
-        unique: "uniqueKiviAdIdPhoto"
-      }
-    },
-    {
-      freezeTableName: true
-    }
-  );
-
-  KiviOriginalAdsPhotos.associate = models => {
-    KiviOriginalAdsPhotos.hasMany(models.KiviOriginal, {
-      foreignKey: "kiviAdId",
-      sourceKey: "kiviAdId",
-      targetKey: "kiviAdId",
-      as: "kiviOriginal"
-    });
-  };
-
-  return KiviOriginalAdsPhotos;
-};

app/models/realEstate.js

@@ -85,9 +85,7 @@ module.exports = (sequelize, DataTypes) => {
     utilityConnection: DataTypes.BOOLEAN,
     distanceToRiver: DataTypes.INTEGER,
     numberOfViewsAgency: DataTypes.INTEGER,
-    numberOfViewsKivi: DataTypes.INTEGER,
-    deleted: DataTypes.BOOLEAN,
-    deletedAt: DataTypes.DATE
+    numberOfViewsKivi: DataTypes.INTEGER
   });
 
   return RealEstate;

app/models/searchRequest.js

@@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [
-          [
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0]
-          ]
-        ],
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -90,7 +82,11 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT
+    heatingType: DataTypes.TEXT,
+    notifiedAt: {
+      type: DataTypes.DATE,
+      defaultValue: new Date()
+    }
   });
 
   return SearchRequest;

app/public/dropzone-5.7.0/.gitignore

@@ -1,7 +0,0 @@
-build
-components
-node_modules
-.DS_Store
-.sass-cache
-_site
-_config.yaml

app/public/dropzone-5.7.0/.tagconfig

@@ -1,52 +0,0 @@
-{
-  "files": [
-    {
-      "name": "src/dropzone.coffee",
-      "regexs": [
-        "Dropzone.version = \"###\""
-      ]
-    },
-    {
-      "name": "dist/dropzone.js",
-      "regexs": [
-        "version = \"###\""
-      ]
-    },
-    {
-      "name": "dist/min/dropzone.min.js",
-      "regexs": [
-        "version=\"###\""
-      ]
-    },
-    {
-      "name": "dist/dropzone-amd-module.js",
-      "regexs": [
-        "version = \"###\""
-      ]
-    },
-    {
-      "name": "dist/min/dropzone-amd-module.min.js",
-      "regexs": [
-        "version=\"###\""
-      ]
-    },
-    {
-      "name": "package.json",
-      "regexs": [
-        "\"version\": \"###\""
-      ]
-    },
-    {
-      "name": "component.json",
-      "regexs": [
-        "\"version\": \"###\""
-      ]
-    },
-    {
-      "name": "bower.json",
-      "regexs": [
-        "\"version\": \"###\""
-      ]
-    }
-  ]
-}

app/public/dropzone-5.7.0/CONTRIBUTING.md

@@ -1,6 +0,0 @@
-Contribute
-==========
-
-DO NOT CREATE PULL REQUESTS ON GITHUB!
-
-I will simply close them. If you want to contribute, please use [gitlab.com](https://gitlab.com/meno/dropzone) instead.

app/public/dropzone-5.7.0/LICENSE

@@ -1,12 +0,0 @@
-License
-
-(The MIT License)
-
-Copyright (c) 2012 Matias Meno <m@tias.me>
-Logo & Website Design (c) 2015 "1910" www.weare1910.com
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

app/public/dropzone-5.7.0/README.md

@@ -1,36 +0,0 @@
-<img alt="Dropzone.js" src="http://www.dropzonejs.com/images/new-logo.svg" />
-
-Dropzone.js is a light weight JavaScript library that turns an HTML element into a dropzone.
-This means that a user can drag and drop a file onto it, and the file gets uploaded to the server via AJAX.
-
-* * *
-
-_If you want support, please use [stackoverflow](http://stackoverflow.com/) with the `dropzone.js` tag and not the
-GitHub issues tracker. Only post an issue here if you think you discovered a bug or have a feature request._
-
-* * *
-
-**Please read the [contributing guidelines](CONTRIBUTING.md) before you start working on Dropzone!**
-
-<br>
-<div align="center">
-  <a href="https://gitlab.com/meno/dropzone/builds/artifacts/master/download?job=release"><strong>&gt;&gt; Download &lt;&lt;</strong></a>
-</div>
-<br>
-<br>
-
-This is no longer the official repository for Dropzone. I have switched to [gitlab.com](https://gitlab.com/meno/dropzone)
-as the primary location to continue development.
- 
-There are multiple reasons why I am switching from GitHub to GitLab, but a few of the reasons are the
-issue tracker that GitHub is providing, *drowning* me in issues that I am unable to categorise or prioritize properly,
-the lack of proper continuous integration, and build files. I don't want the compiled `.js` files in my repository, and
-people regularly commit changes to the compiled files and create pull requests with them.
-
-I will write a blog post soon, that goes into detail about why I am doing the switch.
-
-This repository will still remain, and always host the most up to date versions of dropzone, but only the distribution
-files!
-
-MIT License
------------

app/public/dropzone-5.7.0/bower.json

@@ -1,16 +0,0 @@
-{
-  "name": "dropzone",
-  "location": "enyo/dropzone",
-  "version": "5.7.0",
-  "description": "Dropzone is an easy to use drag'n'drop library. It supports image previews and shows nice progress bars.",
-  "homepage": "http://www.dropzonejs.com",
-  "main": [
-    "dist/min/dropzone.min.css",
-    "dist/min/dropzone.min.js"
-  ],
-  "ignore": [
-    "*",
-    "!dist",
-    "!dist/**/*"
-  ]
-}

app/public/dropzone-5.7.0/component.json

@@ -1,10 +0,0 @@
-{
-  "name": "dropzone",
-  "repo": "enyo/dropzone",
-  "version": "5.7.0",
-  "description": "Handles drag and drop of files for you.",
-  "scripts": [ "index.js", "dist/dropzone.js" ],
-  "styles": [ "dist/basic.css" ],
-  "dependencies": { },
-  "license": "MIT"
-}

app/public/dropzone-5.7.0/composer.json

@@ -1,18 +0,0 @@
-{
-	"name": "enyo/dropzone",
-	"description": "Handles drag and drop of files for you.",
-	"homepage": "http://www.dropzonejs.com",
-	"keywords": [
-		"dragndrop",
-		"drag and drop",
-		"file upload",
-		"upload"
-	],
-	"authors": [{
-		"name": "Matias Meno",
-		"email": "m@tias.me",
-		"homepage": "http://www.matiasmeno.com"
-	}],
-	"license": "MIT",
-	"minimum-stability": "dev"
-}

app/public/dropzone-5.7.0/dist/basic.css

@@ -1,39 +0,0 @@
-/*
- * The MIT License
- * Copyright (c) 2012 Matias Meno <m@tias.me>
- */
-.dropzone, .dropzone * {
-  box-sizing: border-box; }
-
-.dropzone {
-  position: relative; }
-  .dropzone .dz-preview {
-    position: relative;
-    display: inline-block;
-    width: 120px;
-    margin: 0.5em; }
-    .dropzone .dz-preview .dz-progress {
-      display: block;
-      height: 15px;
-      border: 1px solid #aaa; }
-      .dropzone .dz-preview .dz-progress .dz-upload {
-        display: block;
-        height: 100%;
-        width: 0;
-        background: green; }
-    .dropzone .dz-preview .dz-error-message {
-      color: red;
-      display: none; }
-    .dropzone .dz-preview.dz-error .dz-error-message, .dropzone .dz-preview.dz-error .dz-error-mark {
-      display: block; }
-    .dropzone .dz-preview.dz-success .dz-success-mark {
-      display: block; }
-    .dropzone .dz-preview .dz-error-mark, .dropzone .dz-preview .dz-success-mark {
-      position: absolute;
-      display: none;
-      left: 30px;
-      top: 30px;
-      width: 54px;
-      height: 58px;
-      left: 50%;
-      margin-left: -27px; }

app/public/dropzone-5.7.0/dist/dropzone.css

@@ -1,396 +0,0 @@
-/*
- * The MIT License
- * Copyright (c) 2012 Matias Meno <m@tias.me>
- */
-@-webkit-keyframes passing-through {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30%, 70% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); }
-  100% {
-    opacity: 0;
-    -webkit-transform: translateY(-40px);
-    -moz-transform: translateY(-40px);
-    -ms-transform: translateY(-40px);
-    -o-transform: translateY(-40px);
-    transform: translateY(-40px); } }
-@-moz-keyframes passing-through {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30%, 70% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); }
-  100% {
-    opacity: 0;
-    -webkit-transform: translateY(-40px);
-    -moz-transform: translateY(-40px);
-    -ms-transform: translateY(-40px);
-    -o-transform: translateY(-40px);
-    transform: translateY(-40px); } }
-@keyframes passing-through {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30%, 70% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); }
-  100% {
-    opacity: 0;
-    -webkit-transform: translateY(-40px);
-    -moz-transform: translateY(-40px);
-    -ms-transform: translateY(-40px);
-    -o-transform: translateY(-40px);
-    transform: translateY(-40px); } }
-@-webkit-keyframes slide-in {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); } }
-@-moz-keyframes slide-in {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); } }
-@keyframes slide-in {
-  0% {
-    opacity: 0;
-    -webkit-transform: translateY(40px);
-    -moz-transform: translateY(40px);
-    -ms-transform: translateY(40px);
-    -o-transform: translateY(40px);
-    transform: translateY(40px); }
-  30% {
-    opacity: 1;
-    -webkit-transform: translateY(0px);
-    -moz-transform: translateY(0px);
-    -ms-transform: translateY(0px);
-    -o-transform: translateY(0px);
-    transform: translateY(0px); } }
-@-webkit-keyframes pulse {
-  0% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); }
-  10% {
-    -webkit-transform: scale(1.1);
-    -moz-transform: scale(1.1);
-    -ms-transform: scale(1.1);
-    -o-transform: scale(1.1);
-    transform: scale(1.1); }
-  20% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); } }
-@-moz-keyframes pulse {
-  0% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); }
-  10% {
-    -webkit-transform: scale(1.1);
-    -moz-transform: scale(1.1);
-    -ms-transform: scale(1.1);
-    -o-transform: scale(1.1);
-    transform: scale(1.1); }
-  20% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); } }
-@keyframes pulse {
-  0% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); }
-  10% {
-    -webkit-transform: scale(1.1);
-    -moz-transform: scale(1.1);
-    -ms-transform: scale(1.1);
-    -o-transform: scale(1.1);
-    transform: scale(1.1); }
-  20% {
-    -webkit-transform: scale(1);
-    -moz-transform: scale(1);
-    -ms-transform: scale(1);
-    -o-transform: scale(1);
-    transform: scale(1); } }
-.dropzone, .dropzone * {
-  box-sizing: border-box; }
-
-.dropzone {
-  min-height: 150px;
-  border: 2px solid rgba(0, 0, 0, 0.3);
-  background: white;
-  padding: 20px 20px; }
-  .dropzone.dz-clickable {
-    cursor: pointer; }
-    .dropzone.dz-clickable * {
-      cursor: default; }
-    .dropzone.dz-clickable .dz-message, .dropzone.dz-clickable .dz-message * {
-      cursor: pointer; }
-  .dropzone.dz-started .dz-message {
-    display: none; }
-  .dropzone.dz-drag-hover {
-    border-style: solid; }
-    .dropzone.dz-drag-hover .dz-message {
-      opacity: 0.5; }
-  .dropzone .dz-message {
-    text-align: center;
-    margin: 2em 0; }
-    .dropzone .dz-message .dz-button {
-      background: none;
-      color: inherit;
-      border: none;
-      padding: 0;
-      font: inherit;
-      cursor: pointer;
-      outline: inherit; }
-  .dropzone .dz-preview {
-    position: relative;
-    display: inline-block;
-    vertical-align: top;
-    margin: 16px;
-    min-height: 100px; }
-    .dropzone .dz-preview:hover {
-      z-index: 1000; }
-      .dropzone .dz-preview:hover .dz-details {
-        opacity: 1; }
-    .dropzone .dz-preview.dz-file-preview .dz-image {
-      border-radius: 20px;
-      background: #999;
-      background: linear-gradient(to bottom, #eee, #ddd); }
-    .dropzone .dz-preview.dz-file-preview .dz-details {
-      opacity: 1; }
-    .dropzone .dz-preview.dz-image-preview {
-      background: white; }
-      .dropzone .dz-preview.dz-image-preview .dz-details {
-        -webkit-transition: opacity 0.2s linear;
-        -moz-transition: opacity 0.2s linear;
-        -ms-transition: opacity 0.2s linear;
-        -o-transition: opacity 0.2s linear;
-        transition: opacity 0.2s linear; }
-    .dropzone .dz-preview .dz-remove {
-      font-size: 14px;
-      text-align: center;
-      display: block;
-      cursor: pointer;
-      border: none; }
-      .dropzone .dz-preview .dz-remove:hover {
-        text-decoration: underline; }
-    .dropzone .dz-preview:hover .dz-details {
-      opacity: 1; }
-    .dropzone .dz-preview .dz-details {
-      z-index: 20;
-      position: absolute;
-      top: 0;
-      left: 0;
-      opacity: 0;
-      font-size: 13px;
-      min-width: 100%;
-      max-width: 100%;
-      padding: 2em 1em;
-      text-align: center;
-      color: rgba(0, 0, 0, 0.9);
-      line-height: 150%; }
-      .dropzone .dz-preview .dz-details .dz-size {
-        margin-bottom: 1em;
-        font-size: 16px; }
-      .dropzone .dz-preview .dz-details .dz-filename {
-        white-space: nowrap; }
-        .dropzone .dz-preview .dz-details .dz-filename:hover span {
-          border: 1px solid rgba(200, 200, 200, 0.8);
-          background-color: rgba(255, 255, 255, 0.8); }
-        .dropzone .dz-preview .dz-details .dz-filename:not(:hover) {
-          overflow: hidden;
-          text-overflow: ellipsis; }
-          .dropzone .dz-preview .dz-details .dz-filename:not(:hover) span {
-            border: 1px solid transparent; }
-      .dropzone .dz-preview .dz-details .dz-filename span, .dropzone .dz-preview .dz-details .dz-size span {
-        background-color: rgba(255, 255, 255, 0.4);
-        padding: 0 0.4em;
-        border-radius: 3px; }
-    .dropzone .dz-preview:hover .dz-image img {
-      -webkit-transform: scale(1.05, 1.05);
-      -moz-transform: scale(1.05, 1.05);
-      -ms-transform: scale(1.05, 1.05);
-      -o-transform: scale(1.05, 1.05);
-      transform: scale(1.05, 1.05);
-      -webkit-filter: blur(8px);
-      filter: blur(8px); }
-    .dropzone .dz-preview .dz-image {
-      border-radius: 20px;
-      overflow: hidden;
-      width: 120px;
-      height: 120px;
-      position: relative;
-      display: block;
-      z-index: 10; }
-      .dropzone .dz-preview .dz-image img {
-        display: block; }
-    .dropzone .dz-preview.dz-success .dz-success-mark {
-      -webkit-animation: passing-through 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -moz-animation: passing-through 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -ms-animation: passing-through 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -o-animation: passing-through 3s cubic-bezier(0.77, 0, 0.175, 1);
-      animation: passing-through 3s cubic-bezier(0.77, 0, 0.175, 1); }
-    .dropzone .dz-preview.dz-error .dz-error-mark {
-      opacity: 1;
-      -webkit-animation: slide-in 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -moz-animation: slide-in 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -ms-animation: slide-in 3s cubic-bezier(0.77, 0, 0.175, 1);
-      -o-animation: slide-in 3s cubic-bezier(0.77, 0, 0.175, 1);
-      animation: slide-in 3s cubic-bezier(0.77, 0, 0.175, 1); }
-    .dropzone .dz-preview .dz-success-mark, .dropzone .dz-preview .dz-error-mark {
-      pointer-events: none;
-      opacity: 0;
-      z-index: 500;
-      position: absolute;
-      display: block;
-      top: 50%;
-      left: 50%;
-      margin-left: -27px;
-      margin-top: -27px; }
-      .dropzone .dz-preview .dz-success-mark svg, .dropzone .dz-preview .dz-error-mark svg {
-        display: block;
-        width: 54px;
-        height: 54px; }
-    .dropzone .dz-preview.dz-processing .dz-progress {
-      opacity: 1;
-      -webkit-transition: all 0.2s linear;
-      -moz-transition: all 0.2s linear;
-      -ms-transition: all 0.2s linear;
-      -o-transition: all 0.2s linear;
-      transition: all 0.2s linear; }
-    .dropzone .dz-preview.dz-complete .dz-progress {
-      opacity: 0;
-      -webkit-transition: opacity 0.4s ease-in;
-      -moz-transition: opacity 0.4s ease-in;
-      -ms-transition: opacity 0.4s ease-in;
-      -o-transition: opacity 0.4s ease-in;
-      transition: opacity 0.4s ease-in; }
-    .dropzone .dz-preview:not(.dz-processing) .dz-progress {
-      -webkit-animation: pulse 6s ease infinite;
-      -moz-animation: pulse 6s ease infinite;
-      -ms-animation: pulse 6s ease infinite;
-      -o-animation: pulse 6s ease infinite;
-      animation: pulse 6s ease infinite; }
-    .dropzone .dz-preview .dz-progress {
-      opacity: 1;
-      z-index: 1000;
-      pointer-events: none;
-      position: absolute;
-      height: 16px;
-      left: 50%;
-      top: 50%;
-      margin-top: -8px;
-      width: 80px;
-      margin-left: -40px;
-      background: rgba(255, 255, 255, 0.9);
-      -webkit-transform: scale(1);
-      border-radius: 8px;
-      overflow: hidden; }
-      .dropzone .dz-preview .dz-progress .dz-upload {
-        background: #333;
-        background: linear-gradient(to bottom, #666, #444);
-        position: absolute;
-        top: 0;
-        left: 0;
-        bottom: 0;
-        width: 0;
-        -webkit-transition: width 300ms ease-in-out;
-        -moz-transition: width 300ms ease-in-out;
-        -ms-transition: width 300ms ease-in-out;
-        -o-transition: width 300ms ease-in-out;
-        transition: width 300ms ease-in-out; }
-    .dropzone .dz-preview.dz-error .dz-error-message {
-      display: block; }
-    .dropzone .dz-preview.dz-error:hover .dz-error-message {
-      opacity: 1;
-      pointer-events: auto; }
-    .dropzone .dz-preview .dz-error-message {
-      pointer-events: none;
-      z-index: 1000;
-      position: absolute;
-      display: block;
-      display: none;
-      opacity: 0;
-      -webkit-transition: opacity 0.3s ease;
-      -moz-transition: opacity 0.3s ease;
-      -ms-transition: opacity 0.3s ease;
-      -o-transition: opacity 0.3s ease;
-      transition: opacity 0.3s ease;
-      border-radius: 8px;
-      font-size: 13px;
-      top: 130px;
-      left: -10px;
-      width: 140px;
-      background: #be2626;
-      background: linear-gradient(to bottom, #be2626, #a92222);
-      padding: 0.5em 1.2em;
-      color: white; }
-      .dropzone .dz-preview .dz-error-message:after {
-        content: '';
-        position: absolute;
-        top: -6px;
-        left: 64px;
-        width: 0;
-        height: 0;
-        border-left: 6px solid transparent;
-        border-right: 6px solid transparent;
-        border-bottom: 6px solid #be2626; }

app/public/dropzone-5.7.0/dist/min/basic.min.css

@@ -1 +0,0 @@
-.dropzone,.dropzone *{box-sizing:border-box}.dropzone{position:relative}.dropzone .dz-preview{position:relative;display:inline-block;width:120px;margin:0.5em}.dropzone .dz-preview .dz-progress{display:block;height:15px;border:1px solid #aaa}.dropzone .dz-preview .dz-progress .dz-upload{display:block;height:100%;width:0;background:green}.dropzone .dz-preview .dz-error-message{color:red;display:none}.dropzone .dz-preview.dz-error .dz-error-message,.dropzone .dz-preview.dz-error .dz-error-mark{display:block}.dropzone .dz-preview.dz-success .dz-success-mark{display:block}.dropzone .dz-preview .dz-error-mark,.dropzone .dz-preview .dz-success-mark{position:absolute;display:none;left:30px;top:30px;width:54px;height:58px;left:50%;margin-left:-27px}

app/public/dropzone-5.7.0/index.js

@@ -1 +0,0 @@
-module.exports = require("./dist/dropzone.js"); // Exposing dropzone

app/public/dropzone-5.7.0/package.json

@@ -1,40 +0,0 @@
-{
-  "name": "dropzone",
-  "version": "5.7.0",
-  "description": "Handles drag and drop of files for you.",
-  "keywords": [
-    "dragndrop",
-    "drag and drop",
-    "file upload",
-    "upload"
-  ],
-  "homepage": "http://www.dropzonejs.com",
-  "main": "./dist/dropzone.js",
-  "maintainers": [
-    {
-      "name": "Matias Meno",
-      "email": "m@tias.me",
-      "web": "http://www.colorglare.com"
-    }
-  ],
-  "contributors": [
-    {
-      "name": "Matias Meno",
-      "email": "m@tias.me",
-      "web": "http://www.colorglare.com"
-    }
-  ],
-  "scripts": {
-    "test": "grunt && npm run test-prebuilt",
-    "test-prebuilt": "mocha-headless-chrome -f test/test-prebuilt.html -a no-sandbox -a disable-setuid-sandbox"
-  },
-  "bugs": {
-    "email": "m@tias.me",
-    "url": "https://gitlab.com/meno/dropzone/issues"
-  },
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://gitlab.com/meno/dropzone.git"
-  }
-}

app/public/main.css

@@ -158,53 +158,3 @@ h3 {
 .estates-link {
   color: rgba(0, 0, 0, 0.87);
 }
-
-.error {
-  color: #cc0033;
-}
-
-.custom-col {
-  margin-left: auto;
-  left: auto;
-  right: auto;
-}
-
-.dont-break-out {
-  overflow-wrap: break-word;
-  word-wrap: break-word;
-  -ms-word-break: break-all;
-  word-break: break-all;
-  word-break: break-word;
-  -ms-hyphens: auto;
-  -moz-hyphens: auto;
-  -webkit-hyphens: auto;
-  hyphens: auto;
-}
-
-.flex-direction-nav li a {
-  height: 50px;
-}
-
-.slider .slides li {
-  opacity: 1;
-  position: relative;
-}
-.dropzone {
-  background: white;
-  border-radius: 10px;
-  border: 4px dashed #02adba;
-  border-image: none;
-  max-width: 80%;
-  margin-left: auto;
-  margin-right: auto;
-}
-
-.dz-progress {
-  display: none;
-}
-
-.dz-preview .dz-image img {
-  width: 100%;
-  height: 100%;
-  object-fit: cover;
-}

app/routes/index.js

@@ -7,25 +7,13 @@ const {
   getRealEstateTypes,
   postRealEstateTypes
 } = require("../controllers/realEstateTypes");
-const {
-  getPublishTypes,
-  postPublishTypes
-} = require("../controllers/publishRealEstateTypes");
-const {
-  getPublishInputs,
-  postPublishInputs
-} = require("../controllers/publishRealEstate");
-const { getViewRealEstate } = require("../controllers/viewRealEstate");
 const {
   getQueryReview,
   postQueryReview
 } = require("../controllers/queryReview");
 const { getGoAgain } = require("../controllers/goAgain");
-const { publishSuccess } = require("../controllers/publishSuccess");
-const { editSuccess } = require("../controllers/editSuccess");
 const { getLocation, postLocation } = require("../controllers/location");
 const { getUnsubscribe } = require("../controllers/unsubscribe");
-const { getDeletePublishedAd } = require("../controllers/deleteRealEstate");
 const { getRealEstates } = require("../controllers/realEstates");
 const { getRedirect } = require("../controllers/redirect");
 const { getFilters, postFilters } = require("../controllers/realEstateFilters");
@@ -40,16 +28,6 @@ router.get("/vrstanekretnine", getRealEstateTypes);
 router.post("/vrstanekretnine/:searchRequestId", postRealEstateTypes);
 router.post("/vrstanekretnine", postRealEstateTypes);
 
-router.get("/objavinekretninu/:kiviRealEstateId", getPublishTypes);
-router.get("/objavinekretninu", getPublishTypes);
-router.post("/objavinekretninu/:kiviRealEstateId", postPublishTypes);
-router.post("/objavinekretninu", postPublishTypes);
-
-router.get("/podacionekretnini/:kiviRealEstateId", getPublishInputs);
-router.post("/podacionekretnini/:kiviRealEstateId", postPublishInputs);
-
-router.get("/preglednekretnine/:realEstateId", getViewRealEstate);
-
 router.get("/lokacija/:searchRequestId", getLocation);
 router.post("/lokacija/:searchRequestId", postLocation);
 
@@ -61,14 +39,8 @@ router.post("/pregled/:searchRequestId", postQueryReview);
 
 router.get("/odjava/:searchRequestId", getUnsubscribe);
 
-router.get("/obrisioglas/:kiviRealEstateId", getDeletePublishedAd);
-
 router.get("/ponovo", getGoAgain);
 
-router.get("/uspjesnaobjava", publishSuccess);
-
-router.get("/uspjesnaizmjena", editSuccess);
-
 router.get("/nekretnine/:searchRequestId", getRealEstates);
 
 router.get("/redirect/:id", getRedirect);

app/services/notificationService.js

@@ -11,14 +11,14 @@ const {
   generateNotificationEmail,
   generateNewSearchRequestEmail,
   generateEmailSubject,
-  generateCheckUpEmail,
-  generateNewAdPublishEmail
+  generateCheckUpEmail
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
-  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
+
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -36,7 +36,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-
+  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -77,6 +77,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
+
+        //Change time of notified At for searchReq
+        searchRequest.notifiedAt = new Date();
+        searchRequest.save();
       }
     }
   }
@@ -132,7 +136,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -144,42 +148,17 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+
+    //Change time of notified At for searchReq
+    searchRequest.notifiedAt = new Date();
+    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions);
-};
-
-const notifyForNewAdPublish = async (
-  realEstate,
-  kiviOriginal,
-  editingRealEstate
-) => {
-  // console.log("Real estate:", realEstate);
-  // console.log("Kivi original:", kiviOriginal);
-  const email = kiviOriginal.dataValues.email;
-  const emailSubject = editingRealEstate
-    ? `${stagingTag}Kivi - Uspješno ste izmijenili oglas!`
-    : `${stagingTag}Kivi - Uspješno ste objavili oglas!`;
-  const matches = await matchRealEstates([realEstate]);
-  //Counting number of matching req
-  let numberOfMatchingRequests = 0;
-  for (const match in matches) {
-    numberOfMatchingRequests++;
-  }
-
-  const emailContent = generateNewAdPublishEmail(
-    realEstate,
-    kiviOriginal,
-    editingRealEstate,
-    numberOfMatchingRequests
-  );
-
-  await sendEmail(email, emailSubject, emailContent);
+  await Promise.all(asyncSendEmailActions);*/
 };
 
 module.exports = {
   notifyForNewRealEstates,
   notifyForNewSearchRequest,
   notifyRequestsWithDailyOption,
-  notifyForNewAdPublish,
   checkUpNotify
 };

app/services/searchMatchService.js

@@ -9,16 +9,11 @@ const { MAX_REAL_ESTATES_IN_FIRST_EMAIL } = require("../config/appConfig");
 const { EMAIL_FREQUENCY } = require("../common/enums");
 
 const matchRealEstates = async realEstates => {
-  //Filter deleted real estates
-  const filteredRealEstates = realEstates.filter(
-    realEstate => realEstate.deleted === false
-  );
-
-  if (Array.isArray(filteredRealEstates)) {
+  if (Array.isArray(realEstates)) {
     const asyncMatchActions = [];
     const matches = {};
     const matchingRecords = [];
-    for (const realEstate of filteredRealEstates) {
+    for (const realEstate of realEstates) {
       const searchRequestsPromise = findSearchRequestsForRealEstate(realEstate);
       asyncMatchActions.push(searchRequestsPromise);
 

app/views/advancedFilters.ejs

@@ -61,9 +61,8 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
-      <% if (includeIncompleteAds) { %>
            checked
-      <% } %>>
+      >
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

app/views/deleteRealEstate.ejs

@@ -1,12 +0,0 @@
-<!--   -->
-<br><br>
-<div class="row center-align">
-    <img src="../assets/images/logo.svg" alt="kivi logo" width="160">
-</div>
-<div class="row">
-    <div class="col s6 push-s3">
-        <a href="<%= nextStep %>" class="welcome-center-button waves-effect waves-light btn">
-          Početna stranica
-        </a>
-    </div>
-</div>

app/views/editSuccess.ejs

@@ -1,11 +0,0 @@
-<br>
-<div class="row center-align">
-    <p>Vaš oglas je izmijenjen u Kivi bazi.</p>
-    <br>
-    <div class="row center-align">
-        <img src="../assets/images/logo.svg" alt="kivi logo" width="160">
-    </div>
-    <br>
-    <p>Poslali smo potvrdni email sa izmijenjenim detaljima oglasa na Vašu email adresu.</p>
-    <a href="/" class="">Nova pretraga</a>
-</div>

app/views/layout.ejs

@@ -9,28 +9,16 @@
     gtag('js', new Date());
     gtag('config', '<%= process.env.GA_ID %>');
   </script>
- 
+
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.7.0/dropzone.css">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no"/>
 
   <script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
-  <script src="//cdnjs.cloudflare.com/ajax/libs/validate.js/0.13.1/validate.min.js"></script>
-
-  <script src="/assets/dropzone-5.7.0/dist/dropzone.js"></script>
-  <script type="text/javascript">
-    Dropzone.autoDiscover = false;  
-  </script>
-
   <meta charset="UTF-8" />
   <link rel="stylesheet" href="/assets/main.css">
   <link rel="stylesheet" href="/assets/segment.css">
- <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/flexslider/2.7.2/flexslider.css" type="text/css" media="screen" /> 
-
-  <script defer src="https://cdnjs.cloudflare.com/ajax/libs/flexslider/2.7.2/jquery.flexslider.js"></script>
-  
 
   <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
   <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
@@ -59,9 +47,6 @@
     <% } else { %>
     <title>Kivi.ba</title>
   <% } %>
-
-  
-
 </head>
 
 <body>

app/views/location.ejs

@@ -217,7 +217,7 @@
   });
 </script>
 <script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
+  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
   async
   defer
 ></script>

app/views/publishAdditionalData.ejs

@@ -1,50 +0,0 @@
-<br>
-<div class="row">
-<% for (const input of additionalInputInputs){ %>
-    <div class="input-field col s12">
-    <textarea
-            id="<%= input.dbField %>"
-            form="publishForm"
-            name="<%= input.dbField %>"
-            cols="80" rows="15"
-            value="<%= additionalInputValues[input.dbField] !== undefined ? additionalInputValues[input.dbField] : ""%>"
-    ></textarea>
-    <label for="<%= input.dbField %>"><%= input.title %></label>
-</div>
-<% } %>
-</div>
-
-<br>
-<div class="row">
-
-<% for (const input of additionalBooleanPublishInputs){ %>
-  <p class="col s6 m4 l4">
-    <label class="checkbox-label">
-      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
-        <% if (additionalBooleanPublishValues[input.dbField]) { %>
-          checked
-        <% } %>>
-      <span><%= input.title %></span>
-    </label>
-  </p>
-<% } %>
-</div>
-<br>
-<% for (const input of additionalSegmentSelectInputs){ %>
-  <div>
-    <label class="checkbox-label"><%= input.title %>: </label><br><br>
-    <span class="segmented small">
-      <% for (const segmentObject of input.values) { %>
-        <% if (segmentObject.id!=="ANY") { %>
-        <label>
-            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
-            <% if (additionalSegmentSelectValues[input.dbField] === segmentObject.id) { %>
-               checked
-            <% } %>>
-            <span class="label"><%= segmentObject.title %></span>
-        </label>
-        <% } %>
-      <% } %>
-    </span>
-  </div>
-<% } %>

app/views/publishBasicData.ejs

@@ -1,49 +0,0 @@
-<br>
-<div class="row" id="basic-inputs">
-<% for (const input of basicInputInputs){ %>
-    <div class="input-field col s10 m5 l4">
-    <input
-            id="<%= input.dbField %>"
-            name="<%= input.dbField %>"
-            type="text"
-            value="<%= basicInputValues[input.dbField] !== undefined ? basicInputValues[input.dbField] : ""%>"
-    >
-    <label for="<%= input.dbField %>"><%= input.title %></label>
-</div>
-<% } %>
-</div>
-<br>
-<div class="row">
-<% for (const input of basicBooleanPublishInputs){ %>
-  <p>
-    <label class="checkbox-label">
-      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
-        <% if (basicBooleanPublishValues[input.dbField]) { %>
-          checked
-        <% } %>>
-      <span><%= input.title %></span>
-    </label>
-  </p>
-<% } %>
-</div>
-
-<br>
-<% for (const input of basicSegmentSelectInputs){ %>
-  <div>
-    <label class="checkbox-label"><%= input.title %>: </label><br><br>
-    <span class="segmented small">
-      <% for (const segmentObject of input.values) { %>
-        <label>
-            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
-            <% if (basicSegmentSelectValues[input.dbField] === segmentObject.id) { %>
-               checked
-            <% } %>>
-            <span class="label"><%= segmentObject.title %></span>
-        </label>
-      <% } %>
-    </span>
-  </div>
-<% } %>
-
-
-  

app/views/publishEnd.ejs

@@ -1,32 +0,0 @@
-<br>
-<div class="row center-align">
-    <h3>Vaš oglas je spreman!</h3> 
-    Unesite kontakt email i objavite oglas.
-
-<br>
-<div class="row center-align input-field col s3 m4 l5 form-group">
-  <input
-          id="email"
-          name="email"
-          type="email"
-          value="<%= email !== undefined ? email : ""%>"
-  >
-  <div class="messages"></div>
-  <label for="email">Email</label>
-</div>
-</div>
-<br>
-<div class="row center-align">
-    <div class="col s6 push-s3">
-      <a id="submit" href="#" form="publishForm" class="welcome-center-button waves-effect waves-light btn">
-       <% if(editingRealEstate) { %>
-        Snimi izmjene
-      <% } else { %>
-        Objavi oglas
-      <% } %>
-      
-      </a>
-    </div>
-  </div>
-
-<br>

app/views/publishLocation.ejs

@@ -1,224 +0,0 @@
-<div class="row center-align">
-    <h3>
-        Izaberite lokaciju nekretnine na mapi. 
-    </h3>
-  </div>
-  
-  <div class="row center-align">
-    <div class="col s12 m12 l12 xl12">
-      <input
-        id="autocompleteInput"
-        placeholder="Unesite grad, naselje ili ulicu..."
-        type="text"
-      />
-    </div>
-  </div>
-  
-  <div class="row center-align">
-    <div class="col s12">
-      <div id="map"></div>
-    </div>
-  </div>
-  
-    <input type="hidden" name="north" id="north" />
-    <input type="hidden" name="south" id="south" />
-    <input type="hidden" name="east" id="east" />
-    <input type="hidden" name="west" id="west" />
-    <input type="hidden" name="lat" id="lat" />
-    <input type="hidden" name="lng" id="lng" />
-
-    <input type="hidden" name="locationInput" id="locationInput" />
-    <input type="hidden" name="locationInputData" id="locationInputData" />
-  
-  <script>
-    let autocomplete;
-    let map;
-    let places;
-    let geocoder;
-    let marker =false; //Initialy no marker on map
-
-    const editingRealEstate = <%- editingRealEstate %>;
-
-  
-    function locateMe() {
-      if (navigator.geolocation) {
-        function onLocationSuccess(position) {
-          const coordinates =
-            position && position.coords ? position.coords : null;
-          if (coordinates) {
-            const longitude = coordinates.longitude || null;
-            const latitude = coordinates.latitude || null;
-  
-            if (longitude && latitude && map) {
-              map.setCenter({ lat: latitude, lng: longitude });
-              map.setZoom(16);
-            }
-          }
-        }
-  
-        navigator.geolocation.getCurrentPosition(onLocationSuccess);
-      }
-    }
-  
-    function initMap() {
-      const BOSNIA_BOUNDS = {
-        north: 45.7,
-        south: 41.69,
-        west: 15.55,
-        east: 20.77
-      };
-      const SARAJEVO_COORDINATES = {
-        lat: 43.85,
-        lng: 18.41
-      };
-  
-      const mapElement = document.getElementById("map");
-      const restrictMapPanningToBosniaOnly = {
-        latLngBounds: BOSNIA_BOUNDS,
-        strictBounds: true
-      };
-      const initialMapParams = {
-        center: SARAJEVO_COORDINATES,
-        zoom: 12,
-        restriction: restrictMapPanningToBosniaOnly,
-        mapTypeControl: false,
-        panControl: false,
-        zoomControl: true,
-        streetViewControl: false
-      };
-      map = new google.maps.Map(mapElement, initialMapParams);
-  
-      const inputElement = document.getElementById("autocompleteInput");
-      const restrictAutocompleteResultsToBosniaOnly = { country: "ba" };
-      const initialAutocompleteParams = {
-        types: ["geocode"],
-        componentRestrictions: restrictAutocompleteResultsToBosniaOnly,
-        fields: ["geometry", "types", "address_components"]
-      };
-  
-      autocomplete = new google.maps.places.Autocomplete(
-        inputElement,
-        initialAutocompleteParams
-      );
-      autocomplete.bindTo("bounds", map);
-      autocomplete.addListener("place_changed", onPlaceChanged);
-      pacSelectFirst(inputElement);
-      addLocateMeButton(map);
-
-      //Move map and marker to already selected position if in editing mode
-      if( editingRealEstate===true) {
-        //console.log('Editujem mapu!');
-        setMarkerToLocation(map, editingRealEstate);
-      }
-
-      //Add event listener to position marker on map
-      google.maps.event.addListener(map, 'click', positionMarker);
-      
-    }
-
-    function positionMarker(event) {
-        let clickedLocation = event.latLng;
-        if(marker === false){
-            marker = new google.maps.Marker({
-                position: clickedLocation,
-                map: map,
-                draggable: true 
-            });
-            //google.maps.event.addListener(marker, 'dragend', function(event){
-            //    markerLocation();
-          //  });
-        } else{
-            marker.setPosition(clickedLocation);
-        }
-    }
-    function setMarkerToLocation(map) {
-      const ESTATE_COORDINATES = ( <%= locationLat %> !==0 && <%= locationLong %> !== 0 ) ?
-      {
-      lat: <%= locationLat %>,
-      lng: <%= locationLong %>
-      }:
-      { lat: 43.85, //Set to Sarajevo if coordinates are not picked
-        lng: 18.41
-      };
-
-      marker = new google.maps.Marker({
-                position: ESTATE_COORDINATES,
-                map: map,
-                draggable: true 
-            });
-    //  google.maps.event.addListener(marker, 'dragend', function(event){
-         //       markerLocation();
-         //   });
-      //Zooming map to current location
-      map.setCenter(ESTATE_COORDINATES);
-      map.setZoom(13);
-    }
-
-    function addLocateMeButton(map) {
-      var parent = document.createElement("div");
-      parent.className = "locate-me-container";
-  
-      var a = document.createElement("a");
-      a.id = "locateMe";
-      a.className = "btn-floating";
-  
-      var i = document.createElement("i");
-      i.innerText = "gps_fixed";
-      i.className = "material-icons right";
-  
-      a.appendChild(i);
-      a.addEventListener("click", locateMe);
-      parent.appendChild(a);
-  
-      map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent);
-    }
-  
-    function onPlaceChanged() {
-      const place = autocomplete.getPlace();
-      if (place.geometry) {
-        map.fitBounds(place.geometry.viewport);
-        map.setZoom(map.getZoom() + 1);
-        $("#locationInputData").val(JSON.stringify(place));
-      }
-    }
-  
-    function pacSelectFirst(input) {
-      // store the original event binding function
-      const _addEventListener = input.addEventListener
-        ? input.addEventListener
-        : input.attachEvent;
-  
-      function addEventListenerWrapper(type, listener) {
-        // Simulate a 'down arrow' keypress on hitting 'return' when no pac suggestion is selected,
-        // and then trigger the original listener.
-        if (type == "keydown") {
-          const originalListener = listener;
-          listener = function(event) {
-            const suggestionSelected = $(".pac-item-selected").length > 0;
-            if (event.key == "Enter" && !suggestionSelected) {
-              const simulatedDownArrow = $.Event("keydown", {
-                keyCode: 40,
-                which: 40
-              });
-              originalListener.apply(input, [simulatedDownArrow]);
-            }
-  
-            originalListener.apply(input, [event]);
-          };
-        }
-  
-        _addEventListener.apply(input, [type, listener]);
-      }
-  
-      input.addEventListener = addEventListenerWrapper;
-      input.attachEvent = addEventListenerWrapper;
-    }
-      
-  
-  </script>
-  <script
-    src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
-    async
-    defer
-  ></script>
-  

app/views/publishPhotos.ejs

@@ -1,12 +0,0 @@
-<br>
-
-<div action="/photos-upload" class="dropzone" id="photos-upload">
-      <div class="fallback">
-        <input name="file" type="file" multiple />
-      </div>
-      
-    </div>
-
-<br>
-
-<input type="hidden" name="imageUrls" id="imageUrls" value="">

app/views/publishRealEstate.ejs

@@ -1,349 +0,0 @@
-<br>
-
-<form id="publishForm" method="POST" novalidate >
-  <div class="row">
-    <div class="col s12">
-      <ul class="tabs">
-        <li class="tab col s3"><a href="#publishBasicData">Osnovni podaci</a></li>
-        <li class="tab col s3"><a href="#publishAdditionalData">Dodatni podaci</a></li>
-        <li class="tab col s2"><a href="#publishLocation">Lokacija</a></li>
-        <li class="tab col s2"><a href="#publishPhotos">Fotografije</a></li>
-        <li class="tab col s2"><a href="#publishEnd">Kraj</a></li>
-
-    </ul>
-    </div>
-    <div id="publishBasicData" class="col s12">
-      <%- include("./publishBasicData.ejs") %>
-    </div>
-    <div id="publishAdditionalData" class="col s12">
-      <%- include("./publishAdditionalData.ejs") %>
-    </div>
-    <div id="publishLocation" class="col s12">
-     <%- include("./publishLocation.ejs") %>
-    </div>
-    <div id="publishPhotos" class="col s12">
-      <%- include("./publishPhotos.ejs") %>
-     </div>
-    <div id="publishEnd" class="col s12">
-     <%- include("./publishEnd.ejs") %>
-    </div>
-  </div>
-  <input type="hidden" name="editingRealEstate" id="editingRealEstate" value="">
-
-  
-</form>
-
-<script>
-  function uuidv4() {
-      return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-        const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
-        return v.toString(16);
-      });
-    }
-
-    function getFileName(fileName) {
-        const encodedFileName = (uuidv4() + fileName).replace(/\s+/g, '');
-        return encodedFileName;
-    }
-
-    function upload() {
-      var file = $('#selector')[0].files[0];
-      uploadFile(file)
-    }
-
-
-    async function generateSignedURL(file) {
-      const fileName = getFileName(file.name);
-      const response = await fetch('/generateSignedURL?filename=' + fileName);
-      if (!response.ok) {
-        throw new Error('Network response for fetch was not ok.');
-      }
-      let signedUrl = await response.text();
-      signedUrl = signedUrl.replace(/\"/g, "")
-      await uploadFile(file, fileName, signedUrl);
-      return fileName;
-    }
-
-       function uploadFile(file, fileName, url) {
-    return  fetch(url, {
-        method: 'PUT',
-        headers: new Headers({'content-type': 'image/*'}),
-        mode: 'cors',
-        body: file
-      })
-        .then(response => response.text())
-        .then (response => {
-          return response;
-        }
-        )
-        .catch(error => $("#status").html(error)
-        )
-        .then(response => {
-          $("#imageUrls").val($("#imageUrls").val()+ fileName+"|");
-        });
-
-    }
-
-  $(document).ready(function(){
-    $('.tabs').tabs();
-
-    const editingRealEstate = <%- JSON.stringify(editingRealEstate) %>;
-    $("#editingRealEstate").val(editingRealEstate);
-    
-    const currentRealEstatePhotosUrls = <%-JSON.stringify(realEstatePhotosUrls)%>;
-
-    // Manual dropzone init
-    const dropzoneOptions = {
-					url: "/photos-upload", //can be a function that returns url ?
-          autoProcessQueue:false, //not to upload files automaticly
-          method: "put", //or post
-          parallelUploads: 1,
-          uploadMultiple: false, 
-					addRemoveLinks: true,
-					maxFilesize: 2, //MB,
-          resizeWidth: 600, 
-          maxFiles: 10, 
-          acceptedFiles: "image/*",
-        	dictDefaultMessage: `<span class="text-center">
-                              <h3>Prevuci fotografije ili klikni za dodavanje!</h3>
-                              (Maksimalno 10 fotografija.)
-                              </span>`,
-					dictResponseError: 'Error uploading file!',
-          dictRemoveFile: 'Izbriši ',
-          dictFileTooBig: 'Fajl je prevelik!',
-          dictInvalidFileType: 'Iabrani fajl nije fotografija!',
-          dictMaxFilesExceeded: 'Dostigli ste maksimalan broj fotografija!',
-          init: function () {
-            let fileCountOnServer = currentRealEstatePhotosUrls.length; // The number of files already uploaded
-            this.options.maxFiles = this.options.maxFiles - fileCountOnServer;
-
-            if (editingRealEstate) {
-              for (let i=0; i<currentRealEstatePhotosUrls.length; i++) {
-               let fileName = currentRealEstatePhotosUrls[i].replace("https://storage.cloud.google.com/marketalarm-photos/", "");
-               var mockFile = { name: fileName, size: 12345, type: 'image/jpeg', accepted: true, status: Dropzone.ACCEPTED };       
-              this.options.addedfile.call(this, mockFile);
-             this.options.thumbnail.call(this, mockFile, currentRealEstatePhotosUrls[i]);
-             this.files.push(mockFile);
-             mockFile.previewElement.classList.add('dz-success');
-             mockFile.previewElement.classList.add('dz-complete');
-              }
-          };
-
-          this.on("addedfile", function(event) {
-           while (this.files.length > this.options.maxFiles) {
-            this.removeFile(this.files[0]); //automaticly removing if more then 10 files
-          }
-          });  
-
-          this.on("removedfile", function(event) {
-            let fileCountOnPreview = this.files.length;
-            this.options.maxFiles = 10; //resets allowed number of files
-            this._updateMaxFilesReachedClass();
-           });
-          }
-
-        }
-      
-      var photosUploader = new Dropzone('#photos-upload', dropzoneOptions);
-
-
-    //VALIDATION - WiP
-    //Helper validation functions
-    const isValidEmail = $email => {
-      const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
-      return $email && $email.length < 250 && simpleEmailRegex.test($email);
-      };
-
-    const isPresent = $input => {
-      return $input && $input!=="" && $input != null;
-      };
-
-    const isNumber = $input => {
-      const simpleNumberRegex = /([0-9]+[.|,][0-9])|([0-9][.|,][0-9]+)|([0-9]+)/;
-      return $input && $input.length <250 && simpleNumberRegex.test($input) && !isNaN($input);
-      };
-
-    const isInteger = $input => {
-      const simpleIntegerRegex = /^([+-]?[1-9]\d*|0)$/;
-      return $input && $input.length <250 && simpleIntegerRegex.test($input);
-      };
-
-    const form = document.querySelector("#publishForm");
-
-    function showErrors(form, errors) {
-    // We loop through all the inputs and show the errors for that input
-    //_.each(form.querySelectorAll("input[name], select[name]"), function(input) {
-      // Since the errors can be null if no errors were found we need to handle
-      // that
-      showErrorsForInput(input, errors && errors[input.name]);
-   // });
-  }
-
-  // Shows the errors for a specific input
-  function showErrorsForInput(input, errors) {
-    // This is the root of the input
-    var formGroup = closestParent(input.parentNode, "form-group"),
-      // Find where the error messages will be insert into
-      messages = formGroup.querySelector(".messages");
-    // First we remove any old messages and resets the classes
-    resetFormGroup(formGroup);
-    // If we have errors
-    if (errors) {
-      // we first mark the group has having errors
-      formGroup.classList.add("has-error");
-      // then we append all the errors
-      $.each(errors, function(error) {
-        addError(messages, errors[error]);
-      });
-
-    } else {
-      // otherwise we simply mark it as success
-      formGroup.classList.add("has-success");
-    }
-  }
-
-  // Recusively finds the closest parent that has the specified class
-  function closestParent(child, className) {
-    if (!child || child == document) {
-      return null;
-    }
-    if (child.classList.contains(className)) {
-      return child;
-    } else {
-      return closestParent(child.parentNode, className);
-    }
-  }
-
-  function resetFormGroup(formGroup) {
-    formGroup.classList.remove("has-error");
-    formGroup.classList.remove("has-success");
-    $.each(formGroup.querySelectorAll(".help-block.error"), function(el) {
-      el.parentNode.removeChild(el);
-    });
-  }
-
-  // Adds the specified error with the following markup
-  // <p class="help-block error">[message]</p>
-  function addError(messages, error) {
-    var block = document.createElement("p");
-    block.classList.add("help-block");
-    block.classList.add("error");
-    block.innerText = error;
-    messages.appendChild(block);
-  }
-
-  const validate = (input) => {
-
-    let valid=true;
-    let errorMsg =[];
-    let constraint = input.constraint[0];
-
-    switch (constraint) {
-      case "required":
-      valid = isPresent ($(`#${input.dbField}`).val());
-      errorMsg = ["Ovo je obavezno polje."];
-      break;
-      case "numerical":
-      valid = isNumber ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite brojcanu vrijednost."];
-      break;
-      case "integer":
-      valid = isInteger ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite cijeli broj."];
-      break;
-      default :
-      valid = true;
-     }
-    if (!valid) {
-    const inputField = document.querySelector(`#${input.dbField}`);
-    showErrorsForInput( inputField, errorMsg);
-    return false;
-    } else {
-    return true;
-    }
-
-}
-
-    $("#submit").click( async function (e) {
-      e.preventDefault();
-
-      if (marker) {
-        const currentLocation = marker.getPosition();
-
-        $("#lat").val(currentLocation.lat());
-
-        $("#lng").val(currentLocation.lng());
-
-        $("#locationInput").val(
-          document.getElementById("autocompleteInput").value
-          );
-      } else {
-        $("#lat").val(0);
-
-        $("#lng").val(0);
-      }
-      
-       //Tag for checking of error presence 
-       let hasErrors = false;
-      //Check if email is valid
-       const validEmail = isValidEmail($("#email").val());
-       //Show messeges for invalid email is present
-      if (!validEmail) {
-        const errorMsgs = ["Unesite validan email."];
-        const email = document.querySelector("#email");
-        showErrorsForInput( email, errorMsgs) 
-        hasErrors = true;
-      };
-      //Check if other input fields are valid - vratiti se na ovo!!
-      //const basicInputInputs= document.getElementById("basic-inputs").getElementsByTagName("input");
-      
-      //alert(JSON.stringify(""));
-      /*
-      $.each(basicInputInputs, function (input) {
-        alert(input);
-        validate (input);
-      })
-      for (const input of  basicInputInputs ) {
-        alert(input.getAttribute(name));
-        validate (input);
-      } */
-
-      //Filter all files to exclude ones that are already uploaded during the ad publish
-      //But keep them stored to know that they are not deleted from ad
-      let filesForUpload =[];
-      if(editingRealEstate) {
-        let currentPhotosFilenames = currentRealEstatePhotosUrls.map( url => { 
-        return url.replace("https://storage.cloud.google.com/marketalarm-photos/", "");
-         })
-      
-      photosUploader.files.map( file => {
-        if ( currentPhotosFilenames.includes(file.name) ) {
-          return $("#imageUrls").val($("#imageUrls").val()+ file.name+"|");
-        } else if (file.status!=="error") {
-         return  filesForUpload.push(file);
-        }
-      });
-
-      } else {
-        photosUploader.files.map( file => { 
-          if (file.status!=="error") {
-         return  filesForUpload.push(file);
-        }
-        })
-      }
-      
-
-        const asyncUpload =[];
-        filesForUpload.forEach( file => {
-          asyncUpload.push(generateSignedURL(file)); 
-        })
-
-        if (!hasErrors) {
-            await Promise.all(asyncUpload);
-            $("#publishForm").submit();
-      };
-  
-    });
-  });
- 
-</script>

app/views/publishSuccess.ejs

@@ -1,11 +0,0 @@
-<br>
-<div class="row center-align">
-    <p>Vaš oglas je spašen u Kivi bazu.</p>
-    <br>
-    <div class="row center-align">
-        <img src="../assets/images/logo.svg" alt="kivi logo" width="160">
-    </div>
-    <br>
-    <p>Poslali smo potvrdni email sa detaljima oglasa na Vašu email adresu.</p>
-    <a href="/" class="">Nova pretraga</a>
-</div>

app/views/realEstateType.ejs

@@ -9,7 +9,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= labelAdType[0] %></span>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
                 </label>
 
                 <label>
@@ -17,7 +17,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= labelAdType[1] %></span>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
                 </label>
 		    </span>
         </div>

app/views/viewRealEstate.ejs

@@ -1,129 +0,0 @@
-<br/>
-<div class="row col s12 center-align">
-  <div class="col s6 center-align distinguished">
-    <div><%= showAdType %> </div>
-  </div>
-  <div class="col s6 center-align distinguished">
-    <%= showRealEstateType %> 
-  </div>
-</div>
-
- <section class="slider">
-  <div class="flexslider" >
-    <ul class="slides">
-      <% for (const photoUrl of realEstatePhotosUrls) { %>
-      <li class="flex-li">
-        <img src=<%= photoUrl %> alt=""/>
-    </li>
-    <% } %>
-    </ul>
-  </div>
-</section> 
-
-<br/>
-<br>
-<div class="row col s12">
-<% for (const field of inputFields){ %>
-  <p>
-      <span class="col s4"><%= field.title %></span>
-      <span class="col s8 distinguished dont-break-out"><%= allInputValues[field.dbField] %></span>
-  </p>
-  <br>
-<% } %>
-</div>
-
-<br>
-<div class="row">
-<% for (const field of segmentFields){ %>
-  <p>
-      <span class="col s4"><%= field.title.replace(/>/g,'') %></span>
-      <% for (const segmentObject of field.values) { %>
-        <% if (allSegmentSelectedValues[field.dbField] === segmentObject.id) { %>
-                <span class="col s8 distinguished"><%= segmentObject.title.replace(/>/g,'') %></span>
-            <% } %>
-      <% } %>
-  </p>
-  <br>
-<% } %>
-</div>
-<br>
-<div class="row col s12">
-<% for (const field of booleanFields){ %>
-  <p class="col s4">
-    <span>&#10003;</span>
-      <span><%= field.title %></span>
-    
-  </p>
-<% } %>
-</div>
-
-<div class="row center-align ">
-  <div class="distinguished"> 
-    <span>Lokacija nekretnine</span>
-    
-  </div>
-  <br>
-    <br>
-  <div class="col s12">
-    <div id="map"></div>
-  </div>
-</div>
-
-<script type="text/javascript">
-       $(window).load(function() {
-           $('.flexslider').flexslider({
-           animation: "slide",
-           smoothHeight: true
-          });
-        });
-</script>
-
-<script>
-  //Setting up image gallery - carousel
-  
-
-  //Setting up location map
-  let map;
-
-  function initMap() {
-    const BOSNIA_BOUNDS = {
-      north: 45.7,
-      south: 41.69,
-      west: 15.55,
-      east: 20.77
-    };
-    const ESTATE_COORDINATES = {
-      lat: <%= locationLat %>,
-      lng: <%= locationLong %>
-    };
-
-    const mapElement = document.getElementById("map");
-    const restrictMapPanningToBosniaOnly = {
-      latLngBounds: BOSNIA_BOUNDS,
-      strictBounds: true
-    };
-    const initialMapParams = {
-      center: ESTATE_COORDINATES,
-      zoom: 13,
-      restriction: restrictMapPanningToBosniaOnly,
-      mapTypeControl: false,
-      panControl: false,
-      zoomControl: true,
-      streetViewControl: false
-    };
-    map = new google.maps.Map(mapElement, initialMapParams);
-    
-    marker = new google.maps.Marker({
-                position: ESTATE_COORDINATES,
-                map: map,
-            });
-   
-  }
-
-
-</script>
-<script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
-  async
-  defer
-></script>

app/views/welcome.ejs

@@ -18,20 +18,7 @@
     </div>
     <input type="hidden" id="adType" name="adType">
 </form>
-<div class="row center-align">
-    <div>Objavite svoj oglas.</div>
-</div>
-<form method="POST" name="welcomePublishForm">
-    <div class="row center-align">
-        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
-            <a href="#" onclick="publishSaleClick()" class="welcome-center-button btn">Prodaj</a>
-        </div>
-        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
-            <a href="#" onclick="publishRentClick()" class="welcome-center-button btn">Iznajmi</a>
-        </div>
-    </div>
-    <input type="hidden" id="publishAdType" name="publishAdType">
-</form>
+
 <script>
     function saleClick(){
         $("#adType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
@@ -42,13 +29,4 @@
         $("#adType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
         document.welcomeForm.submit();
     }
-    function publishSaleClick(){
-        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
-        document.welcomePublishForm.submit();
-    }
-
-    function publishRentClick(){
-        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
-        document.welcomePublishForm.submit();
-    }
 </script>

development.env

@@ -8,8 +8,9 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-SETTINGS=Variable to denote development, staging and production
+ENVIRONMENT=Variable to denote development, staging and production
 
+USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
@@ -19,10 +20,12 @@ CHECK_UP_DAYS=Check up email is sent after this number of days without notificat
 GA_ID=Google Analytics ID
 
 #=============== GOOGLE MAPS =============#
-GOOGLE_MAP_KEY=(your-key-here)
+API_MAP_KEY=(your-key-here)
 
-#=============== GOOGLE STORAGE =============#
-GOOGLE_APPLICATION_CREDENTIALS="Path to json key file"
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api
+SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
@@ -69,7 +72,8 @@ AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==SALJIC NEKRETNINE==
+SALJIC_MAX_PAGES=Restrict crawler to this number of pages
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

help.js

@@ -1,184 +0,0 @@
-let autocomplete;
-    let map;
-    let places;
-    let geocoder;
-    let marker =false; //Initialy no marker on map
-
-    const editingRealEstate = <%- editingRealEstate %>;
-
-  
-    function locateMe() {
-      if (navigator.geolocation) {
-        function onLocationSuccess(position) {
-          const coordinates =
-            position && position.coords ? position.coords : null;
-          if (coordinates) {
-            const longitude = coordinates.longitude || null;
-            const latitude = coordinates.latitude || null;
-  
-            if (longitude && latitude && map) {
-              map.setCenter({ lat: latitude, lng: longitude });
-              map.setZoom(16);
-            }
-          }
-        }
-  
-        navigator.geolocation.getCurrentPosition(onLocationSuccess);
-      }
-    }
-  
-    function initMap() {
-      const BOSNIA_BOUNDS = {
-        north: 45.7,
-        south: 41.69,
-        west: 15.55,
-        east: 20.77
-      };
-      const SARAJEVO_COORDINATES = {
-        lat: 43.85,
-        lng: 18.41
-      };
-  
-      const mapElement = document.getElementById("map");
-      const restrictMapPanningToBosniaOnly = {
-        latLngBounds: BOSNIA_BOUNDS,
-        strictBounds: true
-      };
-      const initialMapParams = {
-        center: SARAJEVO_COORDINATES,
-        zoom: 12,
-        restriction: restrictMapPanningToBosniaOnly,
-        mapTypeControl: false,
-        panControl: false,
-        zoomControl: true,
-        streetViewControl: false
-      };
-      map = new google.maps.Map(mapElement, initialMapParams);
-  
-      const inputElement = document.getElementById("autocompleteInput");
-      const restrictAutocompleteResultsToBosniaOnly = { country: "ba" };
-      const initialAutocompleteParams = {
-        types: ["geocode"],
-        componentRestrictions: restrictAutocompleteResultsToBosniaOnly,
-        fields: ["geometry", "types", "address_components"]
-      };
-  
-      autocomplete = new google.maps.places.Autocomplete(
-        inputElement,
-        initialAutocompleteParams
-      );
-      autocomplete.bindTo("bounds", map);
-      autocomplete.addListener("place_changed", onPlaceChanged);
-      pacSelectFirst(inputElement);
-      addLocateMeButton(map);
-
-      //Move map and marker to already selected position if in editing mode
-      if( editingRealEstate===true) {
-        console.log('Editujem mapu!');
-        setMarkerToLocation(map, editingRealEstate);
-      }
-
-      //Add event listener to position marker on map
-      google.maps.event.addListener(map, 'click', positionMarker);
-      
-    }
-
-    function positionMarker(event) {
-        let clickedLocation = event.latLng;
-        if(marker === false){
-            marker = new google.maps.Marker({
-                position: clickedLocation,
-                map: map,
-                draggable: true 
-            });
-            //google.maps.event.addListener(marker, 'dragend', function(event){
-            //    markerLocation();
-          //  });
-        } else{
-            marker.setPosition(clickedLocation);
-        }
-    }
-    function setMarkerToLocation(map) {
-      const ESTATE_COORDINATES = ( <%= locationLat %> !==0 && <%= locationLong %> !== 0 ) ?
-      {
-      lat: <%= locationLat %>,
-      lng: <%= locationLong %>
-      }:
-      { lat: 43.85, //Set to Sarajevo if coordinates are not picked
-        lng: 18.41
-      };
-
-      marker = new google.maps.Marker({
-                position: ESTATE_COORDINATES,
-                map: map,
-                draggable: true 
-            });
-    //  google.maps.event.addListener(marker, 'dragend', function(event){
-         //       markerLocation();
-         //   });
-      //Zooming map to current location
-      map.setCenter(ESTATE_COORDINATES);
-      map.setZoom(13);
-    }
-
-    function addLocateMeButton(map) {
-      var parent = document.createElement("div");
-      parent.className = "locate-me-container";
-  
-      var a = document.createElement("a");
-      a.id = "locateMe";
-      a.className = "btn-floating";
-  
-      var i = document.createElement("i");
-      i.innerText = "gps_fixed";
-      i.className = "material-icons right";
-  
-      a.appendChild(i);
-      a.addEventListener("click", locateMe);
-      parent.appendChild(a);
-  
-      map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent);
-    }
-  
-    function onPlaceChanged() {
-      const place = autocomplete.getPlace();
-      if (place.geometry) {
-        map.fitBounds(place.geometry.viewport);
-        map.setZoom(map.getZoom() + 1);
-        $("#locationInputData").val(JSON.stringify(place));
-      }
-    }
-  
-    function pacSelectFirst(input) {
-      // store the original event binding function
-      const _addEventListener = input.addEventListener
-        ? input.addEventListener
-        : input.attachEvent;
-  
-      function addEventListenerWrapper(type, listener) {
-        // Simulate a 'down arrow' keypress on hitting 'return' when no pac suggestion is selected,
-        // and then trigger the original listener.
-        if (type == "keydown") {
-          const originalListener = listener;
-          listener = function(event) {
-            const suggestionSelected = $(".pac-item-selected").length > 0;
-            if (event.key == "Enter" && !suggestionSelected) {
-              const simulatedDownArrow = $.Event("keydown", {
-                keyCode: 40,
-                which: 40
-              });
-              originalListener.apply(input, [simulatedDownArrow]);
-            }
-  
-            originalListener.apply(input, [event]);
-          };
-        }
-  
-        _addEventListener.apply(input, [type, listener]);
-      }
-  
-      input.addEventListener = addEventListenerWrapper;
-      input.attachEvent = addEventListenerWrapper;
-    }
-      
-  

help2.js

@@ -1,112 +0,0 @@
-const validatorFunction = () => {
-  // These are the constraints used to validate the form --just email for now!
-  const constraints = {
-    email: {
-      email: {
-        message: "Proba"
-      },
-      // Email is required
-      presence: true,
-      // and must be an email (duh)
-      email: true
-    }
-  };
-
-  // Hook up the inputs to validate on the fly
-  const inputs = document.querySelectorAll("#email");
-  // inputs.on("change", ev => {
-  //     const errors = validate(form, constraints) || {};
-  //     showErrorsForInput(this, errors[this.name]);
-  //   });
-  // var inputs = document.querySelectorAll("input, textarea, select");
-  for (var i = 0; i < inputs.length; ++i) {
-    inputs.item(i).addEventListener("change", function(ev) {
-      var errors = validate(form, constraints) || {};
-      showErrorsForInput(this, errors[this.name]);
-    });
-  }
-
-  const handleFormSubmit = (form, input) => {
-    // validate the form against the constraints
-    const errors = validate(form, constraints);
-    //
-    console.log("handleFormSubmit error:", errors);
-    // then we update the form to reflect the results
-    showErrors(form, errors || {});
-    if (!errors) {
-      showSuccess();
-    }
-  };
-
-  // Updates the inputs with the validation errors
-  const showErrors = (form, errors) => {
-    // We loop through all the inputs and show the errors for that input
-    $.each(form.querySelectorAll("input[name], select[name]"), input => {
-      // Since the errors can be null if no errors were found we need to handle
-      // that
-      showErrorsForInput(input, errors && errors[input.name]);
-    });
-    //showErrorsForInput(email, errors && errors[email]);
-  };
-
-  // Shows the errors for a specific input
-  const showErrorsForInput = (input, errors) => {
-    // This is the root of the input
-    const formGroup = closestParent(input.parentNode, "form-group"),
-      // Find where the error messages will be insert into
-      messages = formGroup.querySelector(".messages");
-    // First we remove any old messages and resets the classes
-    resetFormGroup(formGroup);
-    // If we have errors
-    if (errors) {
-      //
-      console.log("errors:", errors);
-      // we first mark the group has having errors
-      formGroup.classList.add("has-error");
-      // then we append all the errors
-      $.each(errors, error => {
-        addError(messages, errors[error]);
-      });
-    } else {
-      // otherwise we simply mark it as success
-      formGroup.classList.add("has-success");
-    }
-  };
-
-  // Recusively finds the closest parent that has the specified class
-  const closestParent = (child, className) => {
-    if (!child || child == document) {
-      return null;
-    }
-    if (child.classList.contains(className)) {
-      return child;
-    } else {
-      return closestParent(child.parentNode, className);
-    }
-  };
-
-  const resetFormGroup = formGroup => {
-    // Remove the success and error classes
-    formGroup.classList.remove("has-error");
-    formGroup.classList.remove("has-success");
-    // and remove any old messages
-    $.each(formGroup.querySelectorAll(".help-block.error"), el => {
-      el.parentNode.removeChild(el);
-    });
-  };
-
-  // Adds the specified error with the following markup
-  // <p class="help-block error">[message]</p>
-  const addError = (messages, error) => {
-    const block = document.createElement("p");
-    block.classList.add("help-block");
-    block.classList.add("error");
-    block.innerText = error;
-    messages.appendChild(block);
-  };
-
-  const showSuccess = () => {
-    // We made it \:D/
-    alert("Success!");
-  };
-};

index.js

@@ -4,10 +4,7 @@ const bodyParser = require("body-parser");
 const layout = require("express-layout");
 const compression = require("compression");
 const forceSSL = require("./app/helpers/forceSSL");
-
-const { Storage } = require("@google-cloud/storage");
-const validate = require("validate.js");
-const cors = require("cors");
+const { logDebug } = require("./app/helpers/log");
 
 const {
   APP_PORT,
@@ -23,8 +20,6 @@ const {
 
 const app = express();
 
-app.use(cors());
-
 app.use(forceSSL());
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({ extended: true }));
@@ -44,11 +39,17 @@ app.listen(APP_PORT, () =>
 
 let crawlerRunning = STOP_CRAWLER;
 const crawl = () => {
+  logDebug("Crawl start. crawlerRunning: ", crawlerRunning);
   if (!crawlerRunning) {
     crawlerRunning = true;
     crawlAll().then(newRealEstates => {
-      crawlerRunning = false;
+      logDebug("crawlAll done, new real estate len: ", newRealEstates.length);
       notifyForNewRealEstates(newRealEstates);
+    }).catch(e => {
+      console.error('Error happened: ', e);
+    }).finally(()=> {
+      crawlerRunning = false;
+      logDebug('Finally done!');
     });
   }
 };
@@ -56,36 +57,3 @@ const crawl = () => {
 setInterval(crawl, CRAWLER_INTERVAL * 1000);
 
 setInterval(checkUpNotify, 1000 * 60 * 60 * 24);
-
-//Google storage req
-const PROJECT_ID = "marketalarm";
-const KEY_FILENAME = ""; //relative path
-const BUCKET_NAME = "marketalarm-photos";
-const storage = new Storage();
-
-const bucket = storage.bucket(BUCKET_NAME);
-
-app.get("/generateSignedURL", (req, res) => {
-  async function generateSignedUrl() {
-    // console.log("Started server function!");
-
-    const options = {
-      //Tried to define Google ID and private key while debugging
-      version: "v2", //tried v4 also
-      action: "write",
-      contentType: "image/*", //tried without and with specific image/png ex.
-      expires: Date.now() + 86400000
-    };
-    const filename = req.query.filename;
-
-    // console.log("Filename: ", filename);
-    // console.log("Bucket name:", bucket.name);
-
-    const [url] = await bucket.file(filename).getSignedUrl(options);
-
-    //console.log(`The signed url is ${url}.`);
-
-    res.status(200).send(url);
-  }
-  generateSignedUrl().catch(console.error);
-});

marketalarm-cors.json

@@ -1,12 +0,0 @@
-[
-  {
-    "origin": ["*"],
-    "responseHeader": [
-      "Content-Type",
-      "Access-Control-Allow-Origin",
-      "x-goog-resumable"
-    ],
-    "method": ["GET", "HEAD", "DELETE", "POST", "PUT", "OPTIONS"],
-    "maxAgeSeconds": 3600
-  }
-]

marketalarm-d4b71394407e.json

@@ -1,12 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "marketalarm",
-  "private_key_id": "d4b71394407eb3dba9e431851dab60b198d6985d",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDTE6dkFr0bzDXg\n7ghMxkzq8cajqqqes9JZVqsXh+b/kFJYmEImFUILJJZdI080KM2sEYsIapBCxhMP\nFH017f/gfH3jnRbp3c70hghNh8noSTsq7kPA4l25o8GQnJ6AS+nhy8umPjb4KzX9\nkmC6OOD4P8mAmGqhoUv4s2jld1cXNur6NJjCpjEd2cH3SUbI71oA3V/4W8aK4dvS\n660kLY0PRt7mCiITe0hbTUBZY48W2ijZ7wM2r0HUtPG9XEeGMGmNsC+qD2oWxUU3\nvnm7l1fEIUvLYF4GrLRDJDSkpChBXNcWhoGV2AOvuTc+yghU2+lJWqrKcpLlI23E\nlVJjt9UhAgMBAAECggEABatr8sxq+SQOf9hSIe3Me9Kc1nunrC42scFHRKBNxahJ\ndXw5B9FQPh738Cqhk0xEz6hlrln1Agj6HhRIz8U0r9R+z4TRRr6kfnWmBZAMShu0\nC4JW448abpAYx8CQ/CvRmq2GlF+/M+QBeqpLS8gPzyaKTB/5IBaKG8Bn0fXXQZ2e\n7RaTpGx62jq79omPwiKz0PMVBGZrzPu8Z4tW47muV51osdKSNVgsXb4gCZl28zN2\n6zzY1ZK7u89MesY8joILMHm8cw0oyv9o+RVGEa1I1nq2q1A8ftZny4p7kUA/ITZX\nEZ6SCOP87z9HeVCr8lzexcovD8uZCOTYpcfotlSjGQKBgQD1VVGU2bzExiV0XQGc\n8n6m4TR6Y9zwXBiQPe0rXPZhvsj8QMTXk+L0ejCo4m7NF1dEyH6u+qX6wjNL1Hm9\nN/ZuDFqYtd9w6cQ8CtDZ6QZIE60k6tLQhMNRNMvHdMfedq+VOz3LX6TdyTnv8dP3\nbEsD8wIfFd6t5wNgeZkbKsNxBwKBgQDcQQsUppjglGpUoz7lGHKbFcKMPpIj9fMY\nfze1DXeTAtHGxGm2F10WZvxOEs4DCOUllBlarL5xDAJIJHk/NYlgnI6MJXMLro6d\nsb4iNTUuJKdqAijyOaZQUADJpdYKGwu5y66PUOuojWFV9kiamquXduJ9jzOa1vr1\nSJPUy2YGlwKBgQDNJrpgwa8z0QozAy89Ih68x+fNTMLNkAXOYKp6L3OsixCguDyi\nlP0dOSyFnUvQXutQDmS5R8oSJeElURk4HJsKrXP47WVak3DQUK8S+eSR0zpfe6os\nSkjWGFMriEE2i4MKRI7JCULhX8r+FfgNl9YnCEfG3M/oFhzhyO06JYlncwKBgF6n\nBSAGyEQbA+cDkI/bhcToAQdMDHmvxJyOb147P1vKJmSJG/TI7ZQnBd53blkXhYI0\ntwCko+LpCkH+iqyDUVpXbVsE7P/kMB3MuKzyuLvvvJJuAzK1W6e/+daukeEd5lge\nFBI68EsrFt1eTa1DMuKQkJzs4Xx1TrwCSKV2E45ZAoGBAIZkXyAOhwqCxwDF7B69\nt/7CWs0gPGqp6lFO7fgt7jPmcmSEr/xgUbBDFwd7D49jpXVgCEtr1Bd6MItlu/Ns\nXgXyOa5LPQmglF7UtnvuQLASBy5X6boKaf3sz7I5eho1kXczPGQUHfR5e0DaTND3\nTi2NLIAUci8T7hc8mONdeEHD\n-----END PRIVATE KEY-----\n",
-  "client_email": "marketalarm-photos-service-acc@marketalarm.iam.gserviceaccount.com",
-  "client_id": "115644068453290488813",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/marketalarm-photos-service-acc%40marketalarm.iam.gserviceaccount.com"
-}

package-lock.json

@@ -12,92 +12,6 @@
         "request": "2.x.x"
       }
     },
-    "@google-cloud/common": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-2.4.0.tgz",
-      "integrity": "sha512-zWFjBS35eI9leAHhjfeOYlK5Plcuj/77EzstnrJIZbKgF/nkqjcQuGiMCpzCwOfPyUbz8ZaEOYgbHa759AKbjg==",
-      "requires": {
-        "@google-cloud/projectify": "^1.0.0",
-        "@google-cloud/promisify": "^1.0.0",
-        "arrify": "^2.0.0",
-        "duplexify": "^3.6.0",
-        "ent": "^2.2.0",
-        "extend": "^3.0.2",
-        "google-auth-library": "^5.5.0",
-        "retry-request": "^4.0.0",
-        "teeny-request": "^6.0.0"
-      }
-    },
-    "@google-cloud/paginator": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-2.0.3.tgz",
-      "integrity": "sha512-kp/pkb2p/p0d8/SKUu4mOq8+HGwF8NPzHWkj+VKrIPQPyMRw8deZtrO/OcSiy9C/7bpfU5Txah5ltUNfPkgEXg==",
-      "requires": {
-        "arrify": "^2.0.0",
-        "extend": "^3.0.2"
-      }
-    },
-    "@google-cloud/projectify": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-1.0.4.tgz",
-      "integrity": "sha512-ZdzQUN02eRsmTKfBj9FDL0KNDIFNjBn/d6tHQmA/+FImH5DO6ZV8E7FzxMgAUiVAUq41RFAkb25p1oHOZ8psfg=="
-    },
-    "@google-cloud/promisify": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz",
-      "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ=="
-    },
-    "@google-cloud/storage": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-4.5.0.tgz",
-      "integrity": "sha512-ZLFcR6CiP1AnYBA9eTtASF9Dy3wjYmGx+HZiy/LsIPN41wyBTn9yAjIOxRHiteqzX3uQzZ+VJNCB/DmTU33CeQ==",
-      "requires": {
-        "@google-cloud/common": "^2.1.1",
-        "@google-cloud/paginator": "^2.0.0",
-        "@google-cloud/promisify": "^1.0.0",
-        "arrify": "^2.0.0",
-        "compressible": "^2.0.12",
-        "concat-stream": "^2.0.0",
-        "date-and-time": "^0.12.0",
-        "duplexify": "^3.5.0",
-        "extend": "^3.0.2",
-        "gaxios": "^2.0.1",
-        "gcs-resumable-upload": "^2.2.4",
-        "hash-stream-validation": "^0.2.2",
-        "mime": "^2.2.0",
-        "mime-types": "^2.0.8",
-        "onetime": "^5.1.0",
-        "p-limit": "^2.2.0",
-        "pumpify": "^2.0.0",
-        "readable-stream": "^3.4.0",
-        "snakeize": "^0.1.0",
-        "stream-events": "^1.0.1",
-        "through2": "^3.0.0",
-        "xdg-basedir": "^4.0.0"
-      },
-      "dependencies": {
-        "mime": {
-          "version": "2.4.4",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
-          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
-        },
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        },
-        "xdg-basedir": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz",
-          "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q=="
-        }
-      }
-    },
     "@sendgrid/client": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/@sendgrid/client/-/client-6.3.0.tgz",
@@ -126,10 +40,31 @@
         "@sendgrid/helpers": "^6.3.0"
       }
     },
-    "@tootallnate/once": {
+    "@sozialhelden/fetch-cache": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@sozialhelden/fetch-cache/-/fetch-cache-2.0.1.tgz",
+      "integrity": "sha512-vMlsdT5JQCGjx1fcFxmMNh7ZKppjjsfUAeZEhhNwhEL7GaqbZXsD1OXEyx2IcRa25ZuZtvJSV6Q3rE77VRdLvg==",
+      "requires": {
+        "@sozialhelden/hamster-cache": "^1.0.0"
+      }
+    },
+    "@sozialhelden/hamster-cache": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.0.0.tgz",
-      "integrity": "sha512-KYyTT/T6ALPkIRd2Ge080X/BsXvy9O0hcWTtMWkPvwAwF99+vn6Dv4GzrFT/Nn1LePr+FFDbRXXlqmsy9lw2zA=="
+      "resolved": "https://registry.npmjs.org/@sozialhelden/hamster-cache/-/hamster-cache-1.0.0.tgz",
+      "integrity": "sha512-/TEGA8mdMawZp4Yq/GrkL+72YL5EGuSeVXC3pKW12YY1t3C+zCN/HZ0HRp4zWF/e67svXcxuz/B0AEQxEdvi7A=="
+    },
+    "@supercharge/goodies": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
+      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
+    },
+    "@supercharge/promise-pool": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
+      "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
+      "requires": {
+        "@supercharge/goodies": "~1.4.0"
+      }
     },
     "@types/caseless": {
       "version": "0.12.2",
@@ -187,29 +122,6 @@
         "negotiator": "0.6.1"
       }
     },
-    "agent-base": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz",
-      "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==",
-      "requires": {
-        "debug": "4"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-        }
-      }
-    },
     "ajv": {
       "version": "6.10.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
@@ -298,11 +210,6 @@
       "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
       "dev": true
     },
-    "arrify": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
-      "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug=="
-    },
     "asn1": {
       "version": "0.2.4",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -447,11 +354,6 @@
         "tweetnacl": "^0.14.3"
       }
     },
-    "bignumber.js": {
-      "version": "7.2.1",
-      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
-      "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
-    },
     "binary-extensions": {
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
@@ -556,16 +458,6 @@
         "isarray": "^1.0.0"
       }
     },
-    "buffer-equal-constant-time": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
-      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
-    },
-    "buffer-from": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
-      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
-    },
     "buffer-writer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -813,17 +705,6 @@
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
     },
-    "concat-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
-      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
-      "requires": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^3.0.2",
-        "typedarray": "^0.0.6"
-      }
-    },
     "config-chain": {
       "version": "1.1.12",
       "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz",
@@ -883,15 +764,6 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
-    "cors": {
-      "version": "2.8.5",
-      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
-      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
-      "requires": {
-        "object-assign": "^4",
-        "vary": "^1"
-      }
-    },
     "create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -961,11 +833,6 @@
         "assert-plus": "^1.0.0"
       }
     },
-    "date-and-time": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.12.0.tgz",
-      "integrity": "sha512-n2RJIAp93AucgF/U/Rz5WRS2Hjg5Z+QxscaaMCi6pVZT1JpJKRH+C08vyH/lRR1kxNXnPxgo3lWfd+jCb/UcuQ=="
-    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -1116,41 +983,6 @@
       "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
       "dev": true
     },
-    "duplexify": {
-      "version": "3.7.1",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
-      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
-      "requires": {
-        "end-of-stream": "^1.0.0",
-        "inherits": "^2.0.1",
-        "readable-stream": "^2.0.0",
-        "stream-shift": "^1.0.0"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "2.3.7",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        }
-      }
-    },
     "ecc-jsbn": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1160,14 +992,6 @@
         "safer-buffer": "^2.1.0"
       }
     },
-    "ecdsa-sig-formatter": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
-      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
-      "requires": {
-        "safe-buffer": "^5.0.1"
-      }
-    },
     "editorconfig": {
       "version": "0.15.3",
       "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
@@ -1207,11 +1031,6 @@
         "once": "^1.4.0"
       }
     },
-    "ent": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz",
-      "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0="
-    },
     "entities": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
@@ -1509,11 +1328,6 @@
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
       "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
     },
-    "fast-text-encoding": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz",
-      "integrity": "sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ=="
-    },
     "fill-range": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
@@ -2172,117 +1986,6 @@
         }
       }
     },
-    "gaxios": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.2.tgz",
-      "integrity": "sha512-K/+py7UvKRDaEwEKlLiRKrFr+wjGjsMz5qH7Vs549QJS7cpSCOT/BbWL7pzqECflc46FcNPipjSfB+V1m8PAhw==",
-      "requires": {
-        "abort-controller": "^3.0.0",
-        "extend": "^3.0.2",
-        "https-proxy-agent": "^5.0.0",
-        "is-stream": "^2.0.0",
-        "node-fetch": "^2.3.0"
-      },
-      "dependencies": {
-        "is-stream": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
-          "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
-        }
-      }
-    },
-    "gcp-metadata": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz",
-      "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==",
-      "requires": {
-        "gaxios": "^2.1.0",
-        "json-bigint": "^0.3.0"
-      }
-    },
-    "gcs-resumable-upload": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-2.3.2.tgz",
-      "integrity": "sha512-OPS0iAmPCV+r7PziOIhyxmQOzsazFCy76yYDOS/Z80O/7cuny1KMfqDQa2T0jLaL8EreTU7EMZG5pUuqBKgzHA==",
-      "requires": {
-        "abort-controller": "^3.0.0",
-        "configstore": "^5.0.0",
-        "gaxios": "^2.0.0",
-        "google-auth-library": "^5.0.0",
-        "pumpify": "^2.0.0",
-        "stream-events": "^1.0.4"
-      },
-      "dependencies": {
-        "configstore": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
-          "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==",
-          "requires": {
-            "dot-prop": "^5.2.0",
-            "graceful-fs": "^4.1.2",
-            "make-dir": "^3.0.0",
-            "unique-string": "^2.0.0",
-            "write-file-atomic": "^3.0.0",
-            "xdg-basedir": "^4.0.0"
-          }
-        },
-        "crypto-random-string": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz",
-          "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA=="
-        },
-        "dot-prop": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
-          "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
-          "requires": {
-            "is-obj": "^2.0.0"
-          }
-        },
-        "is-obj": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
-          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
-        },
-        "make-dir": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz",
-          "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==",
-          "requires": {
-            "semver": "^6.0.0"
-          }
-        },
-        "semver": {
-          "version": "6.3.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
-        },
-        "unique-string": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz",
-          "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==",
-          "requires": {
-            "crypto-random-string": "^2.0.0"
-          }
-        },
-        "write-file-atomic": {
-          "version": "3.0.3",
-          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz",
-          "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==",
-          "requires": {
-            "imurmurhash": "^0.1.4",
-            "is-typedarray": "^1.0.0",
-            "signal-exit": "^3.0.2",
-            "typedarray-to-buffer": "^3.1.5"
-          }
-        },
-        "xdg-basedir": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz",
-          "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q=="
-        }
-      }
-    },
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -2353,45 +2056,6 @@
         "ini": "^1.3.4"
       }
     },
-    "google-auth-library": {
-      "version": "5.10.1",
-      "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz",
-      "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==",
-      "requires": {
-        "arrify": "^2.0.0",
-        "base64-js": "^1.3.0",
-        "ecdsa-sig-formatter": "^1.0.11",
-        "fast-text-encoding": "^1.0.0",
-        "gaxios": "^2.1.0",
-        "gcp-metadata": "^3.4.0",
-        "gtoken": "^4.1.0",
-        "jws": "^4.0.0",
-        "lru-cache": "^5.0.0"
-      },
-      "dependencies": {
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
-        "yallist": {
-          "version": "3.1.1",
-          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
-          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
-        }
-      }
-    },
-    "google-p12-pem": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz",
-      "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==",
-      "requires": {
-        "node-forge": "^0.9.0"
-      }
-    },
     "got": {
       "version": "6.7.1",
       "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -2424,24 +2088,6 @@
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
       "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA=="
     },
-    "gtoken": {
-      "version": "4.1.4",
-      "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz",
-      "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==",
-      "requires": {
-        "gaxios": "^2.1.0",
-        "google-p12-pem": "^2.0.0",
-        "jws": "^4.0.0",
-        "mime": "^2.2.0"
-      },
-      "dependencies": {
-        "mime": {
-          "version": "2.4.4",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
-          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
-        }
-      }
-    },
     "har-schema": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
@@ -2493,47 +2139,6 @@
         }
       }
     },
-    "hash-stream-validation": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.2.tgz",
-      "integrity": "sha512-cMlva5CxWZOrlS/cY0C+9qAzesn5srhFA8IT1VPiHc9bWWBLkJfEUIZr7MWoi89oOOGmpg8ymchaOjiArsGu5A==",
-      "requires": {
-        "through2": "^2.0.0"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "2.3.7",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
-          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        },
-        "through2": {
-          "version": "2.0.5",
-          "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
-          "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==",
-          "requires": {
-            "readable-stream": "~2.3.6",
-            "xtend": "~4.0.1"
-          }
-        }
-      }
-    },
     "he": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
@@ -2581,31 +2186,6 @@
         "statuses": ">= 1.4.0 < 2"
       }
     },
-    "http-proxy-agent": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
-      "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==",
-      "requires": {
-        "@tootallnate/once": "1",
-        "agent-base": "6",
-        "debug": "4"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-        }
-      }
-    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -2616,30 +2196,6 @@
         "sshpk": "^1.7.0"
       }
     },
-    "https-proxy-agent": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
-      "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
-      "requires": {
-        "agent-base": "6",
-        "debug": "4"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-        }
-      }
-    },
     "iconv-lite": {
       "version": "0.4.23",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
@@ -2668,7 +2224,8 @@
     "imurmurhash": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o="
+      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+      "dev": true
     },
     "inflection": {
       "version": "1.12.0",
@@ -2962,14 +2519,6 @@
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
       "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
     },
-    "json-bigint": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz",
-      "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=",
-      "requires": {
-        "bignumber.js": "^7.0.0"
-      }
-    },
     "json-schema": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
@@ -3004,25 +2553,6 @@
         "verror": "1.10.0"
       }
     },
-    "jwa": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
-      "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
-      "requires": {
-        "buffer-equal-constant-time": "1.0.1",
-        "ecdsa-sig-formatter": "1.0.11",
-        "safe-buffer": "^5.0.1"
-      }
-    },
-    "jws": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
-      "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
-      "requires": {
-        "jwa": "^2.0.0",
-        "safe-buffer": "^5.0.1"
-      }
-    },
     "kind-of": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
@@ -3310,11 +2840,6 @@
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
       "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
     },
-    "node-forge": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz",
-      "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ=="
-    },
     "node-schedule": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-1.3.2.tgz",
@@ -3401,11 +2926,6 @@
       "resolved": "https://registry.npmjs.org/obj-extend/-/obj-extend-0.1.0.tgz",
       "integrity": "sha1-u0SKR3X7les0p4H5CLusLfI9u1s="
     },
-    "object-assign": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
-    },
     "object-copy": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
@@ -3481,14 +3001,6 @@
         "wrappy": "1"
       }
     },
-    "onetime": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
-      "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
-      "requires": {
-        "mimic-fn": "^2.1.0"
-      }
-    },
     "os-homedir": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
@@ -3745,7 +3257,8 @@
     "process-nextick-args": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
+      "dev": true
     },
     "proto-list": {
       "version": "1.2.4",
@@ -3786,29 +3299,6 @@
         "once": "^1.3.1"
       }
     },
-    "pumpify": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz",
-      "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==",
-      "requires": {
-        "duplexify": "^4.1.1",
-        "inherits": "^2.0.3",
-        "pump": "^3.0.0"
-      },
-      "dependencies": {
-        "duplexify": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz",
-          "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==",
-          "requires": {
-            "end-of-stream": "^1.4.1",
-            "inherits": "^2.0.3",
-            "readable-stream": "^3.1.1",
-            "stream-shift": "^1.0.0"
-          }
-        }
-      }
-    },
     "punycode": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@@ -4041,30 +3531,6 @@
         "any-promise": "^1.3.0"
       }
     },
-    "retry-request": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz",
-      "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==",
-      "requires": {
-        "debug": "^4.1.1",
-        "through2": "^3.0.1"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
-        }
-      }
-    },
     "safe-buffer": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@@ -4162,11 +3628,6 @@
           "version": "6.3.0",
           "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
           "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
-        },
-        "validator": {
-          "version": "10.11.0",
-          "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
-          "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
         }
       }
     },
@@ -4262,11 +3723,6 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
     },
-    "snakeize": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz",
-      "integrity": "sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0="
-    },
     "snapdragon": {
       "version": "0.8.2",
       "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
@@ -4463,19 +3919,6 @@
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
       "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
     },
-    "stream-events": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
-      "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==",
-      "requires": {
-        "stubs": "^3.0.0"
-      }
-    },
-    "stream-shift": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
-      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
-    },
     "string-width": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
@@ -4522,11 +3965,6 @@
       "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
       "dev": true
     },
-    "stubs": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz",
-      "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls="
-    },
     "supports-color": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
@@ -4535,18 +3973,6 @@
         "has-flag": "^3.0.0"
       }
     },
-    "teeny-request": {
-      "version": "6.0.2",
-      "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.2.tgz",
-      "integrity": "sha512-B6fxA0fSnY/bul06NggdN1nywtr5U5Uvt96pHfTi8pi4MNe6++VUWcAAFBrcMeha94s+gULwA5WvagoSZ+AcYg==",
-      "requires": {
-        "http-proxy-agent": "^4.0.0",
-        "https-proxy-agent": "^5.0.0",
-        "node-fetch": "^2.2.0",
-        "stream-events": "^1.0.5",
-        "uuid": "^3.3.2"
-      }
-    },
     "term-size": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz",
@@ -4595,14 +4021,6 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
     },
-    "through2": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz",
-      "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==",
-      "requires": {
-        "readable-stream": "2 || 3"
-      }
-    },
     "timed-out": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",
@@ -4728,19 +4146,6 @@
         "mime-types": "~2.1.18"
       }
     },
-    "typedarray": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
-      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
-    },
-    "typedarray-to-buffer": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
-      "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==",
-      "requires": {
-        "is-typedarray": "^1.0.0"
-      }
-    },
     "umzug": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.2.0.tgz",
@@ -4943,15 +4348,10 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
       "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
     },
-    "validate.js": {
-      "version": "0.13.1",
-      "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz",
-      "integrity": "sha512-PnFM3xiZ+kYmLyTiMgTYmU7ZHkjBZz2/+F0DaALc/uUtVzdCt1wAosvYJ5hFQi/hz8O4zb52FQhHZRC+uVkJ+g=="
-    },
     "validator": {
-      "version": "12.2.0",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-12.2.0.tgz",
-      "integrity": "sha512-jJfE/DW6tIK1Ek8nCfNFqt8Wb3nzMoAbocBF6/Icgg1ZFSBpObdnwVY2jQj6qUqzhx5jc71fpvBWyLGO7Xl+nQ=="
+      "version": "10.11.0",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
+      "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
     },
     "vary": {
       "version": "1.1.2",

package.json

@@ -17,7 +17,8 @@
     "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js",
-    "test-rental-scraper": "cd test && node rentalScrapeTest.js"
+    "test-rental-scraper": "cd test && node rentalScrapeTest.js",
+    "test-saljic-scraper": "cd test && node saljicScrapeTest.js"
   },
   "repository": {
     "type": "git",
@@ -30,13 +31,14 @@
   },
   "dependencies": {
     "2checkout-node": "0.0.1",
-    "@google-cloud/storage": "^4.5.0",
     "@sendgrid/mail": "^6.3.1",
+    "@sozialhelden/fetch-cache": "^2.0.1",
+    "@supercharge/promise-pool": "^1.3.0",
+    "abort-controller": "^3.0.0",
     "aws-sdk": "^2.422.0",
     "bluebird": "^3.5.5",
     "cheerio": "^1.0.0-rc.2",
     "compression": "^1.7.4",
-    "cors": "^2.8.5",
     "dotenv": "^7.0.0",
     "ejs": "^2.6.1",
     "eslint-plugin-prettier": "^3.1.2",
@@ -53,9 +55,7 @@
     "prettier": "^1.19.1",
     "react-step-wizard": "^5.1.0",
     "sequelize": "^5.18.4",
-    "sequelize-cli": "^5.5.0",
-    "validate.js": "^0.13.1",
-    "validator": "^12.2.0"
+    "sequelize-cli": "^5.5.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.0"

test/olxScrapeTest.js

@@ -9,7 +9,7 @@ if (urlToScrape) {
 
   (async () => {
     const data = await crawler.scrapeAd(urlToScrape);
-    console.log(data);
+    console.log("Scraped data:", data);
   })();
 } else {
   console.log("No URL to scrape. Use like this : ");

test/saljicScrapeTest.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const saljicCrawler = require("../app/crawler/specificCrawlers/saljic");
+
+const urlToScrape = process.argv[2] || undefined;
+
+if (urlToScrape) {
+  const crawler = new saljicCrawler();
+
+  (async () => {
+    const data = await crawler.scrapeAd(urlToScrape);
+    console.log("Scraped data:", data);
+  })();
+} else {
+  console.log("No URL to scrape. Use like this : ");
+  console.log("npm run test-saljic-scraper -- URL_TO_SCRAPE");
+}