For revision kivi ads input.

This reverts commit 5f674230e1.

app/common/enums.js

@@ -304,6 +304,7 @@ const AD_AGENCY = {
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
   AKTIDO: "AKTIDO",
+  KIVI: "KIVI",
   SALJIC: "SALJIC"
 };
 

app/common/publishEnums.js

@@ -0,0 +1,492 @@
+const {
+  AD_CATEGORY,
+  ACCESS_ROAD_TYPE,
+  HEATING_TYPE,
+  FURNISHING_TYPE
+} = require("./enums");
+
+const BASIC_BOOLEAN_PUBLISH = [
+  {
+    dbField: "newBuilding",
+    title: "Novogradnja",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "balcony",
+    title: "Balkon",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+
+  {
+    dbField: "elevator",
+    title: "Lift",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "recentlyAdapted",
+    title: "Nedavno adaptirano",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  }
+];
+
+const BASIC_INPUT_PUBLISH = [
+  {
+    dbField: "title",
+    title: "Naslov",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ],
+    constraint: ["required"]
+  },
+  {
+    dbField: "shortDescription",
+    title: "Opis",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ],
+    constraint: []
+  },
+  {
+    dbField: "price",
+    title: "Cijena (KM)",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ],
+    constraint: ["numerical"]
+  },
+  {
+    dbField: "area",
+    title: "Površina (m\xB2)",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ],
+    constraint: ["numerical"]
+  },
+  {
+    dbField: "gardenSize",
+    title: "Površina okućnice (m\xB2)",
+    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE],
+    constraint: ["numerical"]
+  },
+  {
+    dbField: "streetName",
+    title: "Adresa",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ],
+    constraint: []
+  },
+  {
+    dbField: "numberOfRooms",
+    title: "Broj soba",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ],
+    constraint: ["integer"]
+  },
+  {
+    dbField: "numberOfFloors",
+    title: "Broj spratova",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ],
+    constraint: ["integer"]
+  },
+  {
+    dbField: "floor",
+    title: "Sprat",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.OFFICE
+    ],
+    constraint: ["integer"]
+  }
+];
+
+const BASIC_SEGMENT_PUBLISH = [
+  {
+    dbField: "furnishingType",
+    title: "Namještaj",
+    values: Object.keys(FURNISHING_TYPE).map(key => FURNISHING_TYPE[key]),
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  }
+];
+
+const ADDITIONAL_BOOLEAN_PUBLISH = [
+  {
+    dbField: "water",
+    title: "Voda",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "electricity",
+    title: "Struja",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "drainageSystem",
+    title: "Kanalizacija",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "registeredInZkBooks",
+    title: "Uknjiženo",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+
+  {
+    dbField: "parking",
+    title: "Parking",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "garage",
+    title: "Garaža",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "gas",
+    title: "Plin",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "antiTheftDoor",
+    title: "Blindirana vrata",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "airCondition",
+    title: "Klimatizirano",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "phoneConnection",
+    title: "Telefon",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "cableTV",
+    title: "Kablovska",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "internet",
+    title: "Internet",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  },
+  {
+    dbField: "basementAttic",
+    title: "Podrum-Tavan",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+  {
+    dbField: "storeRoom",
+    title: "Ostava",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+  {
+    dbField: "videoSurveillance",
+    title: "Video nadzor",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "alarm",
+    title: "Alarm",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "suitableForStudents",
+    title: "Za studente",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+  {
+    dbField: "includingBills",
+    title: "Uključen trošak režija",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "animalsAllowed",
+    title: "Kućni ljubimci dozvoljeni",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE
+    ]
+  },
+  {
+    dbField: "pool",
+    title: "Bazen",
+    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE]
+  },
+  {
+    dbField: "exchange",
+    title: "Zamjena",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "urbanPlanPermit",
+    title: "Urbanistička dozvola",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "buildingPermit",
+    title: "Građevinska dozvola",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  }
+];
+
+const ADDITIONAL_INPUT_PUBLISH = [
+  {
+    dbField: "longDescription",
+    title: "Detaljan opis",
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  }
+];
+
+const ADDITIONAL_SEGMENT_PUBLISH = [
+  {
+    dbField: "accessRoadType",
+    title: "Pristupni put",
+    values: Object.keys(ACCESS_ROAD_TYPE).map(key => ACCESS_ROAD_TYPE[key]),
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE,
+      AD_CATEGORY.LAND,
+      AD_CATEGORY.GARAGE
+    ]
+  },
+  {
+    dbField: "heatingType",
+    title: "Grijanje",
+    values: Object.keys(HEATING_TYPE).map(key => HEATING_TYPE[key]),
+    categoriesToShow: [
+      AD_CATEGORY.FLAT,
+      AD_CATEGORY.HOUSE,
+      AD_CATEGORY.APARTMENT,
+      AD_CATEGORY.COTTAGE,
+      AD_CATEGORY.OFFICE
+    ]
+  }
+];
+
+module.exports = {
+  BASIC_INPUT_PUBLISH,
+  BASIC_SEGMENT_PUBLISH,
+  BASIC_BOOLEAN_PUBLISH,
+  ADDITIONAL_BOOLEAN_PUBLISH,
+  ADDITIONAL_INPUT_PUBLISH,
+  ADDITIONAL_SEGMENT_PUBLISH
+};

app/config/appConfig.js

@@ -9,7 +9,7 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const STAGING = process.env.ENVIRONMENT !== "production";
+const STAGING = process.env.SETTINGS !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
@@ -41,15 +41,6 @@ const PROSTOR_LOGIN = {
   PASSWORD: process.env.PROSTOR_LOGIN_PASS
 };
 
-const USER_AGENT =
-  process.env.USER_AGENT ||
-  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
-
-const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
-const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
-const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
-const NODE_FETCH_TIMEOUT_MS = parseInt(process.env.NODE_FETCH_TIMEOUT_MS) || 60000
-
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -63,10 +54,6 @@ module.exports = {
   API_MAP_KEY,
   STAGING,
   CHECK_UP_DAYS,
-  PROSTOR_LOGIN,
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY,
-  SCRAPER_API_BASE_URL,
-  NODE_FETCH_TIMEOUT_MS
+  PROSTOR_LOGIN
+
 };

app/controllers/publishRealEstate.js

@@ -0,0 +1,319 @@
+const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
+const { bulkUpsertKiviPhotos } = require("../helpers/db/kiviOriginalAdsPhotos");
+const { currentKiviRealEstate } = require("../helpers/url");
+
+const validate = require("validate.js");
+
+const {
+  AD_CATEGORY,
+  FURNISHING_TYPE,
+  ACCESS_ROAD_TYPE,
+  HEATING_TYPE
+} = require("../common/enums");
+const {
+  BASIC_BOOLEAN_PUBLISH,
+  BASIC_SEGMENT_PUBLISH,
+  ADDITIONAL_BOOLEAN_PUBLISH,
+  ADDITIONAL_SEGMENT_PUBLISH,
+  BASIC_INPUT_PUBLISH,
+  ADDITIONAL_INPUT_PUBLISH
+} = require("../common/publishEnums");
+
+const getPublishInputs = async (req, res) => {
+  const kiviOriginal = await currentKiviRealEstate(req);
+
+  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
+
+  if (!realEstate || !realEstate.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const pageTitle = "Podaci o nekretnini";
+
+  const {
+    price,
+    area,
+    adType,
+    realEstateType,
+    locationLat,
+    locationLong,
+    accessRoadType,
+    heatingType,
+    balcony,
+    newBuilding,
+    elevator,
+    recentlyAdapted,
+    gardenSize,
+    numberOfRooms,
+    numberOfFloors,
+    floor,
+    water,
+    electricity,
+    drainageSystem,
+    registeredInZkBooks,
+    parking,
+    garage,
+    gas,
+    antiTheftDoor,
+    airCondition,
+    phoneConnection,
+    cableTV,
+    internet,
+    basementAttic,
+    storeRoom,
+    videoSurveillance,
+    alarm,
+    suitableForStudents,
+    includingBills,
+    animalsAllowed,
+    pool,
+    exchange,
+    urbanPlanPermit,
+    buildingPermit,
+    furnishingType,
+    shortDescription,
+    streetName,
+    title,
+    longDescription
+  } = realEstate;
+  const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
+
+  // TODO: Maybe this is slow, pay attention to this
+  const filterInputs = filterObject => {
+    const filterCategories = filterObject.categoriesToShow;
+    return filterCategories.indexOf(category) !== -1;
+  };
+  //Boolean inputs to be shown on Basic Data tab
+  const basicBooleanPublishInputs = BASIC_BOOLEAN_PUBLISH.filter(filterInputs);
+  const basicBooleanPublishValues = {
+    balcony,
+    elevator,
+    newBuilding,
+    recentlyAdapted
+  };
+  //Boolean inputs to be shown on Additional Data tab
+  const additionalBooleanPublishInputs = ADDITIONAL_BOOLEAN_PUBLISH.filter(
+    filterInputs
+  );
+  const additionalBooleanPublishValues = {
+    water,
+    electricity,
+    drainageSystem,
+    registeredInZkBooks,
+    parking,
+    garage,
+    gas,
+    antiTheftDoor,
+    airCondition,
+    phoneConnection,
+    cableTV,
+    internet,
+    basementAttic,
+    storeRoom,
+    videoSurveillance,
+    alarm,
+    suitableForStudents,
+    includingBills,
+    animalsAllowed,
+    pool,
+    exchange,
+    urbanPlanPermit,
+    buildingPermit
+  };
+  //Segment select inputs to be shown on Basic Data tab
+  const basicSegmentSelectInputs = BASIC_SEGMENT_PUBLISH.filter(filterInputs);
+  const basicSegmentSelectValues = {
+    furnishingType
+  };
+  //Segment select inputs to be shown on Additional Data tab
+  const additionalSegmentSelectInputs = ADDITIONAL_SEGMENT_PUBLISH.filter(
+    filterInputs
+  );
+  const additionalSegmentSelectValues = {
+    accessRoadType,
+    heatingType
+  };
+  //Input text type inputs to be shown on Basic Data tab
+  const basicInputInputs = BASIC_INPUT_PUBLISH.filter(filterInputs);
+  const basicInputValues = {
+    price,
+    area,
+    gardenSize,
+    numberOfRooms,
+    numberOfFloors,
+    floor,
+    title,
+    shortDescription,
+    streetName
+  };
+  //Input type textare to be shown on Additional Data
+  const additionalInputInputs = ADDITIONAL_INPUT_PUBLISH.filter(filterInputs);
+  const additionalInputValues = {
+    longDescription
+  };
+
+  res.render("publishRealEstate", {
+    title: pageTitle,
+    basicBooleanPublishInputs,
+    basicBooleanPublishValues,
+    additionalBooleanPublishInputs,
+    additionalBooleanPublishValues,
+    basicSegmentSelectInputs,
+    basicSegmentSelectValues,
+    additionalSegmentSelectInputs,
+    additionalSegmentSelectValues,
+    basicInputInputs,
+    basicInputValues,
+    additionalInputInputs,
+    additionalInputValues,
+    validate: validate
+  });
+};
+
+const postPublishInputs = async (req, res) => {
+  const kiviOriginal = await currentKiviRealEstate(req);
+
+  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
+
+  if (!realEstate || !realEstate.dataValues) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const nextStepPage = req.query.nextStep || "/uspjesnaobjava";
+
+  //Request body
+  //console.log("Body:", req.body);
+
+  const balcony = req.body.balcony === "on";
+  const elevator = req.body.elevator === "on";
+  const newBuilding = req.body.newBuilding === "on";
+  const recentlyAdapted = req.body.recentlyAdapted === "on";
+  const water = req.body.water === "on";
+  const electricity = req.body.electricity === "on";
+  const drainageSystem = req.body.drainageSystem === "on";
+  const registeredInZkBooks = req.body.registeredInZkBooks === "on";
+  const parking = req.body.parking === "on";
+  const garage = req.body.garage === "on";
+  const gas = req.body.gas === "on";
+  const antiTheftDoor = req.body.antiTheftDoor === "on";
+  const airCondition = req.body.airCondition === "on";
+  const phoneConnection = req.body.phoneConnection === "on";
+  const cableTV = req.body.cableTV === "on";
+  const internet = req.body.internet === "on";
+  const basementAttic = req.body.basementAttic === "on";
+  const storeRoom = req.body.storeRoom === "on";
+  const videoSurveillance = req.body.videoSurveillance === "on";
+  const alarm = req.body.alarm === "on";
+  const suitableForStudents = req.body.suitableForStudents === "on";
+  const includingBills = req.body.includingBills === "on";
+  const animalsAllowed = req.body.animalsAllowed === "on";
+  const pool = req.body.pool === "on";
+  const exchange = req.body.exchange === "on";
+  const urbanPlanPermit = req.body.urbanPlanPermit === "on";
+  const buildingPermit = req.body.buildingPermit === "on";
+
+  const furnishingType = req.body.furnishingType;
+  //VALIDACIJA TAKO POTVRDITI DA JE ISPRAVNA VRIJEDNOST
+  /* if (!FURNISHING_TYPE[furnishingType]) {
+    res.render("notFound", { title: " Greška !" });
+    return;
+  } */
+  const accessRoadType = req.body.accessRoadType;
+  /*if (!ACCESS_ROAD_TYPE[accessRoadType]) {
+    res.render("notFound", { title: " Greška !" });
+    return;
+  } */
+  const heatingType = req.body.heatingType;
+  /*if (!HEATING_TYPE[heatingType]) {
+    res.render("notFound", { title: " Greška !" });
+    return;
+  }*/
+
+  const price = parseFloat(req.body.price) || null;
+  const area = parseFloat(req.body.area) || null;
+  const gardenSize = parseFloat(req.body.gardenSize) || null;
+  const numberOfRooms = parseInt(req.body.numberOfRooms) || null;
+  const numberOfFloors = parseInt(req.body.numberOfFloors) || null;
+  const floor = parseInt(req.body.floor) || null;
+  const title = req.body.title || "";
+  const shortDescription = req.body.shortDescription || "";
+  const streetName = req.body.streetName || "";
+  const longDescription = req.body.longDescription || "";
+
+  const locationLat = req.body.lat || null;
+  const locationLong = req.body.lng || null;
+  //Contact email saved in other table
+  const contactEmail = req.body.email || "";
+  //Image urls are stored in new table
+  const imageUrls =
+    req.body.imageUrls.split("|").filter(url => url !== "") || [];
+  const imageUrlsData = imageUrls.map(url => {
+    return {
+      kiviAdId: kiviOriginal.kiviAdId,
+      photoUrl: url
+    };
+  });
+  const savedImageUrls = await bulkUpsertKiviPhotos(imageUrlsData);
+
+  realEstate.balcony = balcony;
+  realEstate.elevator = elevator;
+  realEstate.newBuilding = newBuilding;
+  realEstate.recentlyAdapted = recentlyAdapted;
+  realEstate.water = water;
+  realEstate.electricity = electricity;
+  realEstate.drainageSystem = drainageSystem;
+  realEstate.registeredInZkBooks = registeredInZkBooks;
+  realEstate.parking = parking;
+  realEstate.garage = garage;
+  realEstate.gas = gas;
+  realEstate.antiTheftDoor = antiTheftDoor;
+  realEstate.airCondition = airCondition;
+  realEstate.phoneConnection = phoneConnection;
+  realEstate.cableTV = cableTV;
+  realEstate.internet = internet;
+  realEstate.basementAttic = basementAttic;
+  realEstate.storeRoom = storeRoom;
+  realEstate.videoSurveillance = videoSurveillance;
+  realEstate.alarm = alarm;
+  realEstate.suitableForStudents = suitableForStudents;
+  realEstate.includingBills = includingBills;
+  realEstate.animalsAllowed = animalsAllowed;
+  realEstate.pool = pool;
+  realEstate.exchange = exchange;
+  realEstate.urbanPlanPermit = urbanPlanPermit;
+  realEstate.buildingPermit = buildingPermit;
+
+  realEstate.furnishingType = furnishingType;
+  realEstate.accessRoadType = accessRoadType;
+  realEstate.heatingType = heatingType;
+
+  realEstate.price = price;
+  realEstate.area = area;
+  realEstate.gardenSize = gardenSize;
+  realEstate.numberOfRooms = numberOfRooms;
+  realEstate.numberOfFloors = numberOfFloors;
+  realEstate.floor = floor;
+  realEstate.title = title;
+  realEstate.shortDescription = shortDescription;
+  realEstate.streetName = streetName;
+
+  realEstate.longDescription = longDescription;
+
+  realEstate.locationLat = locationLat;
+  realEstate.locationLong = locationLong;
+
+  kiviOriginal.email = contactEmail;
+
+  await realEstate.save();
+
+  await kiviOriginal.save();
+
+  res.redirect(nextStepPage);
+};
+
+module.exports = {
+  getPublishInputs,
+  postPublishInputs
+};

app/controllers/publishRealEstateTypes.js

@@ -0,0 +1,101 @@
+const { currentKiviRealEstate } = require("../helpers/url");
+const {
+  createRealEstate,
+  findRealEstateByAgencyId
+} = require("../helpers/db/realEstate");
+const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
+
+const { AD_CATEGORY, AD_TYPE, AD_AGENCY } = require("../common/enums");
+
+const getPublishTypes = async (req, res) => {
+  const kiviOriginal = await currentKiviRealEstate(req);
+
+  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
+
+  const title = "Koju nekretninu nudite?";
+  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
+  const labelAdType = ["Prodaj", "Iznajmi"];
+
+  if (
+    realEstate &&
+    realEstate.adType &&
+    realEstate.adType === AD_TYPE.AD_TYPE_RENT.stringId
+  ) {
+    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
+  }
+  const realEstateTypes = Object.keys(AD_CATEGORY)
+    .map(category => AD_CATEGORY[category])
+    .filter(category => category.title);
+
+  res.render("realEstateType", {
+    selectedAdType,
+    labelAdType,
+    realEstateTypes,
+    title,
+    AD_TYPE
+  });
+};
+
+const postPublishTypes = async (req, res) => {
+  const kiviOriginal = await currentKiviRealEstate(req);
+
+  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
+
+  const adType = parseInt(req.body.adType);
+
+  const adTypeStringIds = {
+    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
+    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
+  };
+
+  const adTypeStringId =
+    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
+
+  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
+    category => !!AD_CATEGORY[category].title
+  );
+
+  const selectedRealEstateType = req.body.realEstateType || null;
+  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
+  const nextStepPage = req.query.nextStep || "podacionekretnini";
+
+  let nextStepUrl = "";
+  if (kiviOriginal && kiviOriginal.kiviAdId && realEstate && realEstate.id) {
+    //
+    nextStepUrl = `/${nextStepPage}/${kiviOriginal.kiviAdId}`;
+
+    realEstate.adType = adTypeStringId;
+    realEstate.realEstateType = selectedRealEstateType;
+
+    await realEstate.save();
+  } else {
+    try {
+      const newKiviOriginal = await createKiviOriginal({
+        email: ""
+      });
+
+      const newRealEstate = await createRealEstate({
+        adType: adTypeStringId,
+        realEstateType: selectedRealEstateType,
+        url: "http://localhost:5000/",
+        originAgencyName: AD_AGENCY.KIVI,
+        agencyObjectId: newKiviOriginal.kiviAdId
+      });
+
+      nextStepUrl = `/${nextStepPage}/${newKiviOriginal.kiviAdId}`;
+    } catch (error) {
+      console.log(error);
+      nextStepUrl = `/`;
+    }
+  }
+  res.redirect(nextStepUrl);
+};
+
+module.exports = {
+  getPublishTypes,
+  postPublishTypes
+};

app/controllers/publishSuccess.js

@@ -0,0 +1,8 @@
+const publishSuccess = async (req, res) => {
+  const title = "Uspjeh!";
+  res.render("publishSuccess", { title });
+};
+
+module.exports = {
+  publishSuccess
+};

app/controllers/realEstateFilters.js

@@ -122,6 +122,8 @@ const getFilters = async (req, res) => {
 };
 
 const postFilters = async (req, res) => {
+  //
+  console.log("postFilters");
   const searchRequest = await currentSearchRequest(req);
 
   if (!searchRequest || !searchRequest.dataValues) {

app/controllers/realEstateTypes.js

@@ -8,6 +8,7 @@ const getRealEstateTypes = async (req, res) => {
 
   const title = "Koju nekretninu tražite?";
   let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
+  const labelAdType = [AD_TYPE.AD_TYPE_SALE.title, AD_TYPE.AD_TYPE_RENT.title];
   if (
     searchRequest &&
     searchRequest.adType &&
@@ -21,6 +22,7 @@ const getRealEstateTypes = async (req, res) => {
 
   res.render("realEstateType", {
     selectedAdType,
+    labelAdType,
     realEstateTypes,
     title,
     AD_TYPE

app/controllers/welcome.js

@@ -1,6 +1,7 @@
 const { createSearchRequest } = require("../helpers/db/searchRequest");
-
-const { AD_TYPE, AD_CATEGORY } = require("../common/enums");
+const { createRealEstate } = require("../helpers/db/realEstate");
+const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
+const { AD_TYPE, AD_CATEGORY, AD_AGENCY } = require("../common/enums");
 
 const getWelcome = (req, res) => {
   res.render("welcome", {
@@ -11,7 +12,54 @@ const getWelcome = (req, res) => {
 
 const postWelcome = async (req, res) => {
   const adType = parseInt(req.body.adType);
+  const publishAdType = parseInt(req.body.publishAdType);
 
+  let nextStepUrl = "";
+
+  if (adType) {
+    const adTypeStringId = getAdTypeString(adType);
+    try {
+      const newSearchRequest = await createSearchRequest({
+        adType: adTypeStringId,
+        realEstateType: AD_CATEGORY.FLAT.id
+      });
+
+      nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
+    } catch (error) {
+      console.log(error);
+      nextStepUrl = `/`;
+    }
+  } else if (publishAdType) {
+    const adTypeStringId = getAdTypeString(publishAdType);
+
+    try {
+      //First we create new Kivi Ad Original object in db then new Real Estate
+      //Problem with id-s
+      const newKiviOriginal = await createKiviOriginal({
+        email: ""
+      });
+
+      const newRealEstate = await createRealEstate({
+        adType: adTypeStringId,
+        realEstateType: AD_CATEGORY.FLAT.id,
+        //Temp variable because of the not null constraints
+        url: "http://localhost:5000/",
+        originAgencyName: AD_AGENCY.KIVI,
+        agencyObjectId: newKiviOriginal.kiviAdId
+      });
+
+      nextStepUrl = `/objavinekretninu/${newKiviOriginal.kiviAdId}`;
+    } catch (error) {
+      console.log(error);
+      nextStepUrl = `/`;
+    }
+  }
+
+  res.redirect(nextStepUrl);
+};
+
+//--- Helper function
+const getAdTypeString = adType => {
   const adTypeStringIds = {
     [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
     [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
@@ -20,20 +68,7 @@ const postWelcome = async (req, res) => {
   const adTypeStringId =
     adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
 
-  let nextStepUrl = "";
-  try {
-    const newSearchRequest = await createSearchRequest({
-      adType: adTypeStringId,
-      realEstateType: AD_CATEGORY.FLAT.id
-    });
-
-    nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
-  } catch (error) {
-    console.log(error);
-    nextStepUrl = `/`;
-  }
-
-  res.redirect(nextStepUrl);
+  return adTypeStringId;
 };
 
 module.exports = {

app/crawler/crawl.js

@@ -10,7 +10,6 @@ const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
 const SaljicCrawler = require("./specificCrawlers/saljic");
-const { logDebug } = require("../helpers/log");
 
 const {
   OLX_CONFIG,
@@ -76,9 +75,7 @@ async function crawlAll() {
 
   for (const crawler of crawlers) {
     try {
-      logDebug('Starting crawler: ', crawler);
       const newRealEstatesFromSingleCrawler = await crawler.crawl();
-      logDebug('Crawler done: ', crawler);
       if (Array.isArray(newRealEstatesFromSingleCrawler)) {
         newRealEstates.push(...newRealEstatesFromSingleCrawler);
       }

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -159,7 +159,7 @@ class AktidoCrawler {
     }
 
     try {
-      const res = await fetch(url, {}, false);
+      const res = await fetch(url);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -202,10 +202,6 @@ class AktidoCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (

app/crawler/specificCrawlers/olx.js

@@ -1,7 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
-const { logDebug } = require("../../helpers/log");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -45,16 +44,6 @@ const OLX_ENUMS = {
 
 const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
 
-const chunk = (array, size = 10) => {
-  let i, j ,temparray;
-  const result = []
-  for (i=0,j=array.length; i<j; i+=size) {
-    temparray = array.slice(i,i+size);
-    result.push(temparray);
-  }
-  return result;
-}
-
 class OlxCrawler {
   constructor(
     savers = [],
@@ -63,7 +52,7 @@ class OlxCrawler {
     maxPages = 1000,
     maxResultsPerPage = 100,
     ignoredUsernames = [],
-    delayBetweenPages = 500
+    delayBetweenPages = 1000
   ) {
     this.savers = savers;
     this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
@@ -76,7 +65,6 @@ class OlxCrawler {
   }
 
   async crawl() {
-    logDebug("Starting OLX crawl");
     const crawlAdCategories = this.crawlerAdCategories;
 
     const newRealEstates = [];
@@ -100,32 +88,14 @@ class OlxCrawler {
         const entries = singlePageResults.entries();
 
         for (const [index, { value: singlePageResult }] of entries) {
-          if (PRINT_CRAWLER_DEBUG) {
-            console.log("================================");
-            console.log("Category Indexer index : ", index);
-          }
-
           if (singlePageResult) {
-            console.log("\tTotal entries          : ", singlePageResult.length)
             const saveResults = await this.saveCrawledResults(singlePageResult);
             const { newRecords, existingRecords } = saveResults;
 
-            if (PRINT_CRAWLER_DEBUG) {
-              console.log("--------------------------");
-              console.log("\tNew record URLs [", newRecords.length, "] :");
-
-              for(const newRecord of newRecords) {
-                console.log("\t\t",newRecord.url);
-              }
-
-              console.log("\t-------------------------");
-              console.log("\tExisting record URLs [", existingRecords.length, "] :");
-            }
-
             newRealEstates.push(...newRecords);
 
             for (const existingRecord of existingRecords) {
-              const { publishedDate, renewedDate, url } = existingRecord;
+              const { publishedDate, renewedDate } = existingRecord;
 
               const publishedDateMoment = moment.utc(publishedDate);
               const renewedDateMoment = moment.utc(renewedDate);
@@ -135,25 +105,13 @@ class OlxCrawler {
                 "minute"
               );
 
-              if (PRINT_CRAWLER_DEBUG) {
-                console.log("\t\t", url);
-                console.log("\t\t\tPublished date         : ", publishedDate);
-                console.log("\t\t\tRenewed date           : ", renewedDate);
-                console.log("\t\t\tIs same (up to minute) : ", stopCrawlingThisCategory);
-              }
-
               if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
                 generatorsToRemove[index] = true;
-                if (PRINT_CRAWLER_DEBUG) {
-                  console.log("\t\t\tStopping this category indexer");
-                }
+                // console.log("\tGenerator ", index + 1, "has no more new ads");
                 break;
               }
             }
           } else {
-            if (PRINT_CRAWLER_DEBUG) {
-              console.log("\tNo more entries in this category, stopping!");
-            }
             //Generator returned undefined, remove this generator from array
             generatorsToRemove[index] = true;
             // console.log("Generator ", index + 1, "has no more pages");
@@ -178,36 +136,31 @@ class OlxCrawler {
   }
 
   async *categoryIndexer(adCategory) {
-    try {
-      let pageToIndex = 1;
+    let pageToIndex = 1;
 
-      const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
-      const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
-      if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
-        while (true) {
-          const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
-          const singlePageResults = await this.indexSinglePage(
-            urlPageToCrawl,
-            this.maxResultsPerPage
-          );
+    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
 
-          await this.sleep(this.delayBetweenPages);
-          if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
-            yield singlePageResults;
-          } else {
-            return undefined;
-          }
-
-          ++pageToIndex;
-          if (pageToIndex === this.maxPages) {
-            return undefined;
-          }
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
         }
-      } else {
-        return undefined;
       }
-    } catch (e) {
-      console.log('Error inside generator: ', e);
+    } else {
+      return undefined;
     }
   }
 
@@ -217,10 +170,8 @@ class OlxCrawler {
     }
 
     try {
-      const res = await fetch(url, {}, false);
-      logDebug("Got category results for: ", url);
+      const res = await fetch(url);
       const body = await res.text();
-      logDebug("Got category results text for: ", url);
       const $ = cheerio.load(body);
       let hrefs = [];
 
@@ -241,44 +192,26 @@ class OlxCrawler {
 
       const asyncScraping = [];
       for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(hrefs[i]);
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
       }
 
-      const allChunks = chunk(asyncScraping, 2);
-      const dataResults = []
-      for (let i = 0; i < allChunks.length; i++) {
-        const singleChunk = allChunks[i];
-        const promises = singleChunk.map(c => this.scrapeAd(c))
-        const chunkResults = await Promise.all(promises);
-        await this.sleep(this.delayBetweenPages);
-        dataResults.push(...chunkResults);
-        logDebug("Chunk results len:", chunkResults.length);
-      }
-
-
-      const filteredScrapedData = dataResults.filter(adData => !!adData);
-      logDebug("Filtered scraped data length: ", filteredScrapedData.length);
-
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
       return filteredScrapedData;
     } catch (e) {
-      console.error("Exception caught, index single page: " + e);
+      console.error("Exception caught:" + e);
       return [];
     }
   }
 
   async scrapeAd(url) {
-    logDebug("Scraping : ", url);
+    // console.log("Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
       let status = AD_STATUS.STATUS_NORMAL;
 
-      if (body.indexOf('<html') === -1) {
-        console.error("This is the body: ", body);
-        throw { message: 'Failed to fetch page !' }
-      }
-
       const propertySelectors = {
         username:
           "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
@@ -305,25 +238,37 @@ class OlxCrawler {
 
       //====== PRICE DETECTION AND EXTRACTION =====
       let price = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      const urgentPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
+      )
+        .text()
+        .trim();
 
-      const priceHeader = $("#pc > p.n").text().trim();
-      const priceValue = $("#pc > p:nth-child(2)").text().trim();
-      price = priceValue;
-
-      if (priceHeader.indexOf('Hitn') !== -1) {
-        // Urgent price
-        status = AD_STATUS.STATUS_URGENT;
-      }
-
-      const discountPriceTag = $("#artikal_glavni_div > div.artikal_lijevo > p:nth-child(4)").text().trim();
-      if (discountPriceTag.indexOf('Akcij') !== -1) {
-        status = AD_STATUS.STATUS_DISCOUNTED;
-        const discountPriceValues = $("#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p").text().trim();
-        // discountPriceValues contain string like "10.000 KM 7.500 KM"
-        // First price is regular, second is currently active (discounted) price
-        const bothPrices = discountPriceValues.split('KM');
-        // Now, currently active price is second element of bothPrices array
-        price = bothPrices[1] ? bothPrices[1].trim() : null;
+      if (normalPriceValue && normalPriceValue.length > 0) {
+        price = normalPriceValue;
+        if (
+          $("#pc > p.n")
+            .text()
+            .indexOf("Hitna") !== -1
+        ) {
+          status = AD_STATUS.STATUS_URGENT;
+        } else {
+          status = AD_STATUS.STATUS_NORMAL;
+        }
+      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.split("KM");
+        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
+        if (priceValues.length > 1) {
+          price = priceValues[1].trim();
+          status = AD_STATUS.STATUS_DISCOUNTED;
+        } else {
+          throw { message: "Can't find urgent price" };
+        }
+      } else {
+        throw {
+          message: "Can't find price (it is not normal nor urgent price ?)"
+        };
       }
 
       //====== OTHER AD INFORMATION ===============
@@ -333,7 +278,7 @@ class OlxCrawler {
 
       let otherInformationDivId;
       //We need to locate DIV ID where other information are stored
-      for (let possibleId = 1; possibleId <= 30; possibleId++) {
+      for (let possibleId = 10; possibleId <= 20; possibleId++) {
         const adTypeFieldTitle = $(
           `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
         )
@@ -705,12 +650,10 @@ class OlxCrawler {
         distanceToRiver,
         numberOfViewsAgency
       };
-      //
-      //console.log("Scraped data:", data);
 
       return data;
     } catch (e) {
-      console.error("Exception caught scrapeAd : " + e.message, "\r\nURL:", url);
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
     return null;
   }
@@ -825,9 +768,6 @@ class OlxCrawler {
     if (!priceText) {
       return NaN;
     }
-    if (priceText === "Po dogovoru") {
-      return null;
-    }
     const formattedPriceText = priceText.replace(".", "").replace(",", ".");
     return parseFloat(formattedPriceText);
   }

app/crawler/specificCrawlers/prostor.js

@@ -1,10 +1,9 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
-const nodeFetch = require("node-fetch");
 
 const {
   AD_TYPE,
@@ -63,19 +62,13 @@ class ProstorCrawler {
 
   async crawl() {
     const crawlAdCategories = this.crawlerAdCategories;
-    const crawlAdTypes = this.crawlerAdTypes;
-    if (!crawlAdCategories || !crawlAdTypes) {
-      return []
-    }
-
-    const newRealEstates = [];
     //We need session cookie to use login privileges
     const prostorCookie = await this.getCookies();
-    //New tag to check if crawler logged in
+    //New tag to check if crawler loged in
     const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
-
+    const newRealEstates = [];
     //Crawl only if login was successful
-    if (login) {
+    if (crawlAdCategories && login) {
       const indexGenerators = [];
       for (const adCategory of crawlAdCategories) {
         indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
@@ -141,11 +134,6 @@ class ProstorCrawler {
         prostorCookie
       );
 
-      if (!Array.isArray(listOfAllRealEstates)){
-        console.log('[PROSTOR] Could not find real estate JSON data, check selector !');
-        return undefined;
-      }
-
       let elementToStartIndexFrom = 0;
       while (true) {
         const realEstatesForSinglePage = listOfAllRealEstates.slice(
@@ -203,22 +191,24 @@ class ProstorCrawler {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
 
     //Status information is given already in realestate list
-    const adStatus = ProstorCrawler.getStatusId(status);
+    //For VIP Ads status ='' canot be used, but no VIP ads are crawled
+    //We will make "fake" vip ad for RE that have size=55
+    //It is weird because yesterday it said 'VIP ponuda' ???
+    const adStatus =
+      size === "55"
+        ? ProstorCrawler.getStatusId("VIP ponuda")
+        : ProstorCrawler.getStatusId(status);
 
     const url = `https://prostor.ba${link}`;
 
     // console.log("[PROSTOR] Scraping : ", url);
     try {
-      const adPageSource = await nodeFetch(url, {
+      const adPageSource = await fetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
       // link contains part of the URL in the format of : /prodaja/stan/stup/9556
       // general form is : /actionType/realEstateType/location/realEstateID
       // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
@@ -443,14 +433,14 @@ class ProstorCrawler {
     }
 
     try {
-      const res = await nodeFetch(url, {
+      const res = await fetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await res.text();
       const $ = cheerio.load(body);
 
       const scriptElement = $(
-          "body > div.content > div.container-fluid > script:nth-child(6)"
+        "body > div > div.container-fluid > script:nth-child(7)"
       );
 
       if (
@@ -607,7 +597,7 @@ class ProstorCrawler {
     formData.append("email", PROSTOR_LOGIN.EMAIL);
     formData.append("password", PROSTOR_LOGIN.PASSWORD);
 
-    return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
+    return fetch("https://prostor.ba/moj-prostor/prijava", {
       method: "POST",
       body: formData,
       headers: { Cookie: prostorCookie }
@@ -634,12 +624,9 @@ class ProstorCrawler {
       });
   }
   async getCookies() {
-    const getResponse = await nodeFetch(
-      "https://prostor.ba/moj-prostor/prijava",
-      {
-        headers: { Cookie: "" }
-      }
-    );
+    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
+      headers: { Cookie: "" }
+    });
     const raw = getResponse.headers.raw()["set-cookie"];
     const cookie = raw
       .map(datastring => {

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -159,7 +159,7 @@ class RentalCrawler {
     }
 
     try {
-      const res = await fetch(url, {} , false);
+      const res = await fetch(url);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -202,10 +202,6 @@ class RentalCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
       const mapElementParent = $(".box-map").parent();
       const scriptElement = $("script", mapElementParent);
       if (
@@ -403,9 +399,7 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${
-              extractedData["re_realEstates_inserted"]
-            }`
+            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
           };
         }
 
@@ -416,9 +410,7 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${
-              extractedData["re_realEstates_edited"]
-            }`
+            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -1,10 +1,8 @@
 "use strict";
 
-const fetch = require("../../helpers/fetchWrapper");
-const { getUrlParams } = require("../../helpers/url");
+const fetch = require("node-fetch");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
-const PromisePool = require('@supercharge/promise-pool');
 
 const {
   AD_TYPE,
@@ -48,13 +46,12 @@ class SaljicCrawler {
     maxPages = 5000,
     maxResultsPerPage = 5000,
     ignoredUsernames = [],
-    delayBetweenPages = 500
+    delayBetweenPages = 1000
   ) {
     this.savers = savers;
     this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
     this.crawlerAdTypes = crawlerAdTypes;
     this.crawlerAdCategories = crawlerAdCategories;
-    this.maxPages = maxPages
     this.maxResultsPerPage = maxResultsPerPage;
     this.delayBetweenPages = delayBetweenPages;
   }
@@ -160,7 +157,7 @@ class SaljicCrawler {
     }
 
     try {
-      const res = await fetch(url, {}, false);
+      const res = await fetch(url);
       const body = await res.text();
       const $ = cheerio.load(body);
       let hrefs = [];
@@ -208,23 +205,11 @@ class SaljicCrawler {
 
       const asyncScraping = [];
       for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push([hrefsAbs[i], adTypes[i]]);
+        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
       }
 
-      
-
-      const dataResults = []
-      const { scrapedData, errors } = await PromisePool
-        .withConcurrency(2)
-        .for(asyncScraping)
-        .process(async data => {
-          const result = await this.scrapeAd(...data)
-          await this.sleep(this.delayBetweenPages);
-          dataResults.push(result)
-          return result; //TODO: this does not work, scrapedData is null, dataResults works
-        })
-
-      const filteredScrapedData = dataResults.filter(adData => !!adData);
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
       return filteredScrapedData;
     } catch (e) {
       console.error("[SALJIC] Exception caught:" + e);
@@ -239,20 +224,10 @@ class SaljicCrawler {
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url
-      const agencyObjectId = url
-        ? parseInt(url.substring(46, url.length))
-        : null;
-
-      if (!agencyObjectId) {
-        throw { message : 'No agency object ID - URL changed?'}
-      }
+      const agencyObjectId = parseInt(url.substring(46, url.length));
 
       //Extracting main properties
       const propertySelectors = {
@@ -262,10 +237,11 @@ class SaljicCrawler {
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
         streetName:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
+
         descriptions:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
         latAndLong:
-          "iframe"
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
       };
       const title = $(propertySelectors.title)
         .text()
@@ -296,26 +272,14 @@ class SaljicCrawler {
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      let latText;
-      let longText;
-      if (latAndLongSrc){
-        const mapParams = getUrlParams(latAndLongSrc);
-        if (mapParams) {
-          if (mapParams['marker']){
-            const marker = mapParams['marker'].split(',');
-            latText = marker[0] ? marker[0] : undefined;
-            longText = marker[1] ? marker[1] : undefined;
-          }else{
-            if (mapParams['mlat']) {
-              latText = mapParams['mlat'];
-            }
-            if (mapParams['mlon']) {
-              longText = mapParams['mlon'];
-            }
-          }
-        }
-      }
-
+      const latText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("marker=") + 7,
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
+      );
+      const longText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
+        latAndLongSrc.length
+      );
       const locationLat = parseFloat(latText) || null;
       const locationLong = parseFloat(longText) || null;
 
@@ -364,10 +328,11 @@ class SaljicCrawler {
       let numberOfViewsKivi = null;
       let streetNumber = 0;
       let adStatus = status;
-      let shortDescription = descriptions
-        ? descriptions.substring(0, descriptions.indexOf("."))
-        : "";
-      let longDescription = descriptions || "";
+      let shortDescription = descriptions.substring(
+        0,
+        descriptions.indexOf(".")
+      );
+      let longDescription = descriptions;
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -378,14 +343,10 @@ class SaljicCrawler {
           .replace(/[\n\r\t]/gm, "")
           .trim();
 
-        const mainFieldTitle = mainField
-          ? mainField.substring(0, mainField.indexOf("  "))
-          : "";
+        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
         const mainFieldValue = mainField
-          ? mainField
-              .substring(mainField.indexOf("  "), mainField.length)
-              .trim()
-          : "";
+          .substring(mainField.indexOf("  "), mainField.length)
+          .trim();
 
         switch (mainFieldTitle) {
           case "Površina":
@@ -402,7 +363,7 @@ class SaljicCrawler {
             numberOfRooms = parseInt(mainFieldValue);
             break;
           case "Broj spratova":
-            numberOfFloors = this.parseNumberOfFloors(mainFieldValue);
+            numberOfFloors = parseInt(mainFieldValue);
             break;
           case "Sprat":
             floor = parseInt(mainFieldValue);
@@ -448,9 +409,6 @@ class SaljicCrawler {
             )
             .trim();
           realEstateType = this.getAdCategoryId(categoryTmp);
-          if (!realEstateType) {
-            throw { message: 'No real estate type - page body not loaded correctly or page changed?' }
-          }
         } else {
           switch (additionalField) {
             case "Internet":
@@ -607,7 +565,7 @@ class SaljicCrawler {
 
       return data;
     } catch (e) {
-      console.error("[SALJIC] Exception caught: " + e.message, "\r\nURL:", url);
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
     return null;
   }
@@ -652,21 +610,6 @@ class SaljicCrawler {
     }
   }
 
-  parseNumberOfFloors(numberOfFloorsText) {
-    const tryNumericalValue = parseInt(numberOfFloorsText);
-    if (!isNaN(tryNumericalValue)){
-      return tryNumericalValue;
-    }
-
-    // Guess number of floors based on number of + sign concatenations
-    // e.g. P+S+Pt -> 3 floors
-    if (typeof numberOfFloorsText === 'string' && numberOfFloorsText.indexOf('+') > 0) {
-      return numberOfFloorsText.split('+').length + 1
-    }
-
-    return null
-  }
-
   async sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }

app/helpers/db/kiviOriginal.js

@@ -0,0 +1,20 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const createKiviOriginal = async (kiviAdFields = {}) => {
+  return await db.KiviOriginal.create(kiviAdFields);
+};
+
+const getKiviOriginalById = async id => {
+  try {
+    return db.KiviOriginal.findByPk(id);
+  } catch (error) {
+    console.log("kiviOriginal.js", error);
+    return null;
+  }
+};
+module.exports = {
+  createKiviOriginal,
+  getKiviOriginalById
+};

app/helpers/db/kiviOriginalAdsPhotos.js

@@ -0,0 +1,17 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const bulkUpsertKiviPhotos = async kiviPhotosData => {
+  try {
+    return await db.KiviOriginalAdsPhotos.bulkCreate(kiviPhotosData, {
+      ignoreDuplicates: true
+    });
+  } catch (e) {
+    console.log("Error bulk upserting kiviOriginalAdsPhotos : ", e);
+  }
+};
+
+module.exports = {
+  bulkUpsertKiviPhotos
+};

app/helpers/db/realEstate.js

@@ -77,9 +77,28 @@ const bulkUpsertRealEstates = async realEstateData => {
 };
 
 const getRealEstateById = async id => {
-  return db.RealEstate.findByPk(id);
+  try {
+    return db.RealEstate.findByPk(id);
+  } catch (error) {
+    console.log("realEstate.js", error);
+    return null;
+  }
 };
 
+const createRealEstate = async (realEstateFields = {}) => {
+  return await db.RealEstate.create(realEstateFields);
+};
+
+const findRealEstateByAgencyId = async kiviId => {
+  try {
+    return db.RealEstate.findOne({
+      where: { agencyObjectId: kiviId }
+    });
+  } catch (error) {
+    console.log("realEstate.js", error);
+    return null;
+  }
+};
 const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const {
     priceMin,
@@ -332,14 +351,10 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where:
-      includeIncompleteAds || includeIncompleteAds == null
-        ? queryIncludeIncomplete
-        : query,
+    where: includeIncompleteAds ? queryIncludeIncomplete : query,
     limit: maxResults,
     order
   });
@@ -348,5 +363,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
 module.exports = {
   bulkUpsertRealEstates,
   getRealEstateById,
-  findRealEstatesForSearchRequest
+  createRealEstate,
+  findRealEstatesForSearchRequest,
+  findRealEstateByAgencyId
 };

app/helpers/db/searchRequest.js

@@ -3,7 +3,6 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -17,22 +16,6 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
-const findAllRequestsForCheckUp = async () => {
-  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
-  const checkupDate = new Date();
-  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
-
-  const dateQuery = {
-    notifiedAt: {
-      [Op.lte]: checkupDate
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: dateQuery
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -174,7 +157,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
+      //Or ones that picked some values but also picked to includeIncomplete ads
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -193,10 +176,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -246,10 +226,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -298,10 +275,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
+              [Op.eq]: true
             }
           }
         ]
@@ -339,10 +313,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -376,10 +347,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -413,10 +381,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
+                  [Op.eq]: true
                 }
               }
             ]
@@ -458,13 +423,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query which is default
+  //Tag to check if incomplete ads are accepted in query
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.or]: {
-        [Op.eq]: true,
-        [Op.is]: null
-      }
+      [Op.eq]: true
     };
   }
 
@@ -476,6 +438,5 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate,
-  findAllRequestsForCheckUp
+  findSearchRequestsForRealEstate
 };

app/helpers/db/searchRequestMatch.js

@@ -2,6 +2,7 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -42,6 +43,42 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
+const findAllRequestsForCheckUp = async () => {
+  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
+  //The ones that received notification for real estate CHECK_UP_DAYS days from now
+  const date = new Date();
+  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
+  date.setDate(checkUpDate);
+  const dateQuery = {
+    createdAt: {
+      [Op.gte]: date
+    }
+  };
+
+  const excludedMatches = await db.SearchRequestMatch.findAll({
+    attributes: ["searchRequestId"],
+    where: dateQuery,
+    order: [["searchRequestId", "ASC"]]
+  });
+
+  const excludedRequestsAll = excludedMatches.map(match => {
+    return match.dataValues.searchRequestId;
+  });
+  //Removing duplicate search request id-s for optimization
+  const excludedRequests = [...new Set(excludedRequestsAll)];
+
+  const query = {
+    subscribed: true,
+    id: {
+      [Op.notIn]: excludedRequests
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: query
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -52,5 +89,6 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp
 };

app/helpers/emailContentGenerator.js

@@ -9,7 +9,6 @@ const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
 //Tag to recognize staging from development
 const stagingTag = STAGING ? "[STAGING] " : "";
-const wordOfMouthRequest = `Molimo vas <strong>recite svojim prijateljima</strong> za Kivi - što više korisnika budemo imali, moći ćemo više agencija uključiti i više nekretnina imati u bazi. Hvala!`
 
 const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
   return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>
@@ -70,9 +69,6 @@ const generateNotificationEmail = (
   ${moreRealEstates}
   </div>
   <br/>
-  ${wordOfMouthRequest}
-  <br/>
-  <br/>
   ${emailFooter}`;
 };
 
@@ -136,10 +132,6 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
   </div>
   ${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
   <br/>
-  <br/>
-  ${wordOfMouthRequest}
-  <br/>
-  <br/>
   ${emailFooter}`;
 };
 

app/helpers/fetchWrapper.js

@@ -1,58 +0,0 @@
-const nodeFetch = require("node-fetch");
-const AbortController = require('abort-controller');
-const FetchCache = require('@sozialhelden/fetch-cache').default;
-
-console.log("Fc ", FetchCache)
-
-const {
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY,
-  SCRAPER_API_BASE_URL,
-  NODE_FETCH_TIMEOUT_MS
-} = require("../config/appConfig");
-
-const timeout = (ms) => {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-const fetchCache = new FetchCache({
-  fetch: nodeFetch,
-  cacheOptions: {
-    // Don't save more than 100 responses in the cache. Allows infinite responses by default
-    maximalItemCount: 10000,
-    // When should the cache evict responses when its full?
-    evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
-    defaultTTL: 6 * 60 * 60 * 1000  // 6 hours
-    // ...see https://github.com/sozialhelden/hamster-cache for all possible options
-  },
-});
-
-
-const fetch = async (url, options = {}, useCache = true) => {
-  const controller = new AbortController();
-
-  const newOptions = Object.assign({}, options);
-  if (!newOptions["headers"]) {
-    newOptions["headers"] = {};
-  }
-
-  newOptions.signal = controller.signal;
-
- // newOptions["headers"]["User-Agent"] = USER_AGENT;
-
-  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
-  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
-    urlToFetchThroughAPI = url; 
-  }
-
-  const urlAdaptedForScraping = USE_SCRAPER_API
-    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
-    : url;
-  const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);
-
-  const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
-  return result;
-};
-
-module.exports = fetch;

app/helpers/log.js

@@ -1,13 +0,0 @@
-const {
-  PRINT_CRAWLER_DEBUG
-} = require("../config/appConfig");
-
-const logDebug = (...args) => {
-  if (PRINT_CRAWLER_DEBUG) {
-    console.log(...args);
-  }
-}
-
-module.exports = {
-  logDebug
-};

app/helpers/url.js

@@ -1,4 +1,6 @@
 const { getSearchRequest } = require("./db/searchRequest");
+const { getRealEstateById } = require("./db/realEstate");
+const { getKiviOriginalById } = require("./db/kiviOriginal");
 
 const currentSearchRequest = async req => {
   const searchRequestId =
@@ -8,25 +10,22 @@ const currentSearchRequest = async req => {
   return await getSearchRequest(searchRequestId);
 };
 
-const getUrlParams = function (url) {
-  if (typeof url === 'string' && url.length > 0){
-    const params = {};
-    const questionMarkIndex = url.indexOf('?');
-    if (questionMarkIndex === -1) {
-      return undefined;
-    }
-    const query = url.substring(questionMarkIndex+1);
-    const vars = query.split('&');
-    for (let i = 0; i < vars.length; i++) {
-      const pair = vars[i].split('=');
-      params[pair[0]] = decodeURIComponent(pair[1]);
-    }
-    return params;
-  }
-  return undefined;
+const currentRealEstate = async req => {
+  const realEstateId = req && req.params ? req.params["realEstateId"] : null;
+  if (!realEstateId) return null;
+
+  return await getRealEstateById(parseInt(realEstateId));
+};
+const currentKiviRealEstate = async req => {
+  const kiviRealEstateId =
+    req && req.params ? req.params["kiviRealEstateId"] : null;
+  if (!kiviRealEstateId) return null;
+
+  return await getKiviOriginalById(kiviRealEstateId);
 };
 
 module.exports = {
   currentSearchRequest,
-  getUrlParams
+  currentRealEstate,
+  currentKiviRealEstate
 };

app/migrations/20200203114630-add-kiviOriginal-table.js

@@ -0,0 +1,28 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      kiviAdId: {
+        type: Sequelize.UUID,
+        defaultValue: Sequelize.UUIDV4,
+        allowNull: false,
+        primaryKey: true
+      },
+      email: Sequelize.TEXT,
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("KiviOriginal", tableFields);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable("KiviOriginal", {});
+  }
+};

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -1,14 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
-      type: Sequelize.DATE,
-      defaultValue: new Date()
-    });
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
-  }
-};

app/migrations/20200312210336-add-table-kivi-photos.js

@@ -0,0 +1,39 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false
+      },
+      kiviAdId: {
+        type: Sequelize.UUID,
+        allowNull: false,
+        references: {
+          model: "KiviOriginal",
+          key: "kiviAdId"
+        }
+      },
+      photoUrl: {
+        type: Sequelize.TEXT,
+        allowNull: false
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("KiviOriginalAdsPhotos", tableFields);
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable("KiviOriginalAdsPhotos", {});
+  }
+};

app/models/index.js

@@ -16,7 +16,7 @@ config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;
 
 let sequelize;
 if (config.use_env_variable) {
-  sequelize = new Sequelize(process.env[config.use_env_variable] + "?ssl=true", config);
+  sequelize = new Sequelize(process.env[config.use_env_variable], config);
 } else {
   sequelize = new Sequelize(
     config.database,

app/models/kiviOriginal.js

@@ -0,0 +1,21 @@
+"use strict";
+
+module.exports = (sequalize, DataTypes) => {
+  const KiviOriginal = sequalize.define(
+    "KiviOriginal",
+    {
+      kiviAdId: {
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4,
+        allowNull: false,
+        primaryKey: true
+      },
+      email: DataTypes.TEXT
+    },
+    {
+      freezeTableName: true
+    }
+  );
+
+  return KiviOriginal;
+};

app/models/kiviOriginalAdsPhotos.js

@@ -0,0 +1,41 @@
+"use strict";
+
+module.exports = (sequalize, DataTypes) => {
+  const KiviOriginalAdsPhotos = sequalize.define(
+    "KiviOriginalAdsPhotos",
+    {
+      id: {
+        type: DataTypes.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false
+      },
+      kiviAdId: {
+        type: DataTypes.UUID,
+        allowNull: false,
+        references: {
+          model: "KiviOriginal",
+          key: "kiviAdId"
+        }
+      },
+      photoUrl: {
+        type: DataTypes.TEXT,
+        allowNull: false
+      }
+    },
+    {
+      freezeTableName: true
+    }
+  );
+
+  KiviOriginalAdsPhotos.associate = models => {
+    KiviOriginalAdsPhotos.hasMany(models.KiviOriginal, {
+      foreignKey: "kiviAdId",
+      sourceKey: "kiviAdId",
+      targetKey: "kiviAdId",
+      as: "kiviOriginal"
+    });
+  };
+
+  return KiviOriginalAdsPhotos;
+};

app/models/searchRequest.js

@@ -15,7 +15,15 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+        coordinates: [
+          [
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0]
+          ]
+        ],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -82,11 +90,7 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT,
-    notifiedAt: {
-      type: DataTypes.DATE,
-      defaultValue: new Date()
-    }
+    heatingType: DataTypes.TEXT
   });
 
   return SearchRequest;

app/public/main.css

@@ -158,3 +158,27 @@ h3 {
 .estates-link {
   color: rgba(0, 0, 0, 0.87);
 }
+
+.error {
+  color: #cc0033;
+}
+
+.custom-col {
+  margin-left: auto;
+  left: auto;
+  right: auto;
+}
+
+.dropzone {
+  background: white;
+  border-radius: 10px;
+  border: 4px dashed #02adba;
+  border-image: none;
+  max-width: 80%;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.dz-progress {
+  display: none;
+}

app/routes/index.js

@@ -7,11 +7,20 @@ const {
   getRealEstateTypes,
   postRealEstateTypes
 } = require("../controllers/realEstateTypes");
+const {
+  getPublishTypes,
+  postPublishTypes
+} = require("../controllers/publishRealEstateTypes");
+const {
+  getPublishInputs,
+  postPublishInputs
+} = require("../controllers/publishRealEstate");
 const {
   getQueryReview,
   postQueryReview
 } = require("../controllers/queryReview");
 const { getGoAgain } = require("../controllers/goAgain");
+const { publishSuccess } = require("../controllers/publishSuccess");
 const { getLocation, postLocation } = require("../controllers/location");
 const { getUnsubscribe } = require("../controllers/unsubscribe");
 const { getRealEstates } = require("../controllers/realEstates");
@@ -28,6 +37,14 @@ router.get("/vrstanekretnine", getRealEstateTypes);
 router.post("/vrstanekretnine/:searchRequestId", postRealEstateTypes);
 router.post("/vrstanekretnine", postRealEstateTypes);
 
+router.get("/objavinekretninu/:kiviRealEstateId", getPublishTypes);
+router.get("/objavinekretninu", getPublishTypes);
+router.post("/objavinekretninu/:kiviRealEstateId", postPublishTypes);
+router.post("/objavinekretninu", postPublishTypes);
+
+router.get("/podacionekretnini/:kiviRealEstateId", getPublishInputs);
+router.post("/podacionekretnini/:kiviRealEstateId", postPublishInputs);
+
 router.get("/lokacija/:searchRequestId", getLocation);
 router.post("/lokacija/:searchRequestId", postLocation);
 
@@ -41,6 +58,8 @@ router.get("/odjava/:searchRequestId", getUnsubscribe);
 
 router.get("/ponovo", getGoAgain);
 
+router.get("/uspjesnaobjava", publishSuccess);
+
 router.get("/nekretnine/:searchRequestId", getRealEstates);
 
 router.get("/redirect/:id", getRedirect);

app/services/notificationService.js

@@ -15,10 +15,9 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
+  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
-const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
-
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -36,7 +35,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
+
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -77,10 +76,6 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
-
-        //Change time of notified At for searchReq
-        searchRequest.notifiedAt = new Date();
-        searchRequest.save();
       }
     }
   }
@@ -136,7 +131,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+  const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -148,12 +143,8 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
-
-    //Change time of notified At for searchReq
-    searchRequest.notifiedAt = new Date();
-    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions);*/
+  await Promise.all(asyncSendEmailActions);
 };
 
 module.exports = {

app/views/advancedFilters.ejs

@@ -61,8 +61,9 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
+      <% if (includeIncompleteAds) { %>
            checked
-      >
+      <% } %>>
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

app/views/layout.ejs

@@ -9,13 +9,21 @@
     gtag('js', new Date());
     gtag('config', '<%= process.env.GA_ID %>');
   </script>
-
+ 
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.7.0/dropzone.css">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no"/>
 
   <script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
+  <script src="//cdnjs.cloudflare.com/ajax/libs/validate.js/0.13.1/validate.min.js"></script>
+
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.7.0/dropzone.js"></script>
+  <script type="text/javascript">
+    Dropzone.autoDiscover = false;  
+  </script>
+
   <meta charset="UTF-8" />
   <link rel="stylesheet" href="/assets/main.css">
   <link rel="stylesheet" href="/assets/segment.css">
@@ -47,6 +55,9 @@
     <% } else { %>
     <title>Kivi.ba</title>
   <% } %>
+
+  
+
 </head>
 
 <body>

app/views/publishAdditionalData.ejs

@@ -0,0 +1,50 @@
+<br>
+<div class="row">
+<% for (const input of additionalInputInputs){ %>
+    <div class="input-field col s12">
+    <textarea
+            id="<%= input.dbField %>"
+            form="publishForm"
+            name="<%= input.dbField %>"
+            cols="80" rows="15"
+            value="<%= additionalInputValues[input.dbField] !== undefined ? additionalInputValues[input.dbField] : ""%>"
+    ></textarea>
+    <label for="<%= input.dbField %>"><%= input.title %></label>
+</div>
+<% } %>
+</div>
+
+<br>
+<div class="row">
+
+<% for (const input of additionalBooleanPublishInputs){ %>
+  <p class="col s6 m4 l4">
+    <label class="checkbox-label">
+      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
+        <% if (additionalBooleanPublishValues[input.dbField]) { %>
+          checked
+        <% } %>>
+      <span><%= input.title %></span>
+    </label>
+  </p>
+<% } %>
+</div>
+<br>
+<% for (const input of additionalSegmentSelectInputs){ %>
+  <div>
+    <label class="checkbox-label"><%= input.title %>: </label><br><br>
+    <span class="segmented small">
+      <% for (const segmentObject of input.values) { %>
+        <% if (segmentObject.id!=="ANY") { %>
+        <label>
+            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
+            <% if (additionalSegmentSelectValues[input.dbField] === segmentObject.id) { %>
+               checked
+            <% } %>>
+            <span class="label"><%= segmentObject.title %></span>
+        </label>
+        <% } %>
+      <% } %>
+    </span>
+  </div>
+<% } %>

app/views/publishBasicData.ejs

@@ -0,0 +1,49 @@
+<br>
+<div class="row" id="basic-inputs">
+<% for (const input of basicInputInputs){ %>
+    <div class="input-field col s10 m5 l4">
+    <input
+            id="<%= input.dbField %>"
+            name="<%= input.dbField %>"
+            type="text"
+            value="<%= basicInputValues[input.dbField] !== undefined ? basicInputValues[input.dbField] : ""%>"
+    >
+    <label for="<%= input.dbField %>"><%= input.title %></label>
+</div>
+<% } %>
+</div>
+<br>
+<div class="row">
+<% for (const input of basicBooleanPublishInputs){ %>
+  <p>
+    <label class="checkbox-label">
+      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
+        <% if (basicBooleanPublishValues[input.dbField]) { %>
+          checked
+        <% } %>>
+      <span><%= input.title %></span>
+    </label>
+  </p>
+<% } %>
+</div>
+
+<br>
+<% for (const input of basicSegmentSelectInputs){ %>
+  <div>
+    <label class="checkbox-label"><%= input.title %>: </label><br><br>
+    <span class="segmented small">
+      <% for (const segmentObject of input.values) { %>
+        <label>
+            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
+            <% if (basicSegmentSelectValues[input.dbField] === segmentObject.id) { %>
+               checked
+            <% } %>>
+            <span class="label"><%= segmentObject.title %></span>
+        </label>
+      <% } %>
+    </span>
+  </div>
+<% } %>
+
+
+  

app/views/publishEnd.ejs

@@ -0,0 +1,24 @@
+<br>
+<div class="row center-align">
+    <h3>Vaš oglas je spreman!</h3> 
+    Unesite kontakt email i objavite oglas.
+
+<br>
+<div class="row center-align input-field col s3 m4 l5 form-group">
+  <input
+          id="email"
+          name="email"
+          type="email"
+  >
+  <div class="messages"></div>
+  <label for="email">Email</label>
+</div>
+</div>
+<br>
+<div class="row center-align">
+    <div class="col s6 push-s3">
+      <a id="submit" href="#" form="publishForm" class="welcome-center-button waves-effect waves-light btn">Objavi oglas</a>
+    </div>
+  </div>
+
+<br>

app/views/publishLocation.ejs

@@ -0,0 +1,192 @@
+<div class="row center-align">
+    <h3>
+        Izaberite lokaciju nekretnine na mapi. 
+    </h3>
+  </div>
+  
+  <div class="row center-align">
+    <div class="col s12 m12 l12 xl12">
+      <input
+        id="autocompleteInput"
+        placeholder="Unesite grad, naselje ili ulicu..."
+        type="text"
+      />
+    </div>
+  </div>
+  
+  <div class="row center-align">
+    <div class="col s12">
+      <div id="map"></div>
+    </div>
+  </div>
+  
+    <input type="hidden" name="north" id="north" />
+    <input type="hidden" name="south" id="south" />
+    <input type="hidden" name="east" id="east" />
+    <input type="hidden" name="west" id="west" />
+    <input type="hidden" name="lat" id="lat" />
+    <input type="hidden" name="lng" id="lng" />
+
+    <input type="hidden" name="locationInput" id="locationInput" />
+    <input type="hidden" name="locationInputData" id="locationInputData" />
+  
+  <script>
+    let autocomplete;
+    let map;
+    let places;
+    let geocoder;
+    let marker =false; //Initialy no marker on map
+  
+    function locateMe() {
+      if (navigator.geolocation) {
+        function onLocationSuccess(position) {
+          const coordinates =
+            position && position.coords ? position.coords : null;
+          if (coordinates) {
+            const longitude = coordinates.longitude || null;
+            const latitude = coordinates.latitude || null;
+  
+            if (longitude && latitude && map) {
+              map.setCenter({ lat: latitude, lng: longitude });
+              map.setZoom(16);
+            }
+          }
+        }
+  
+        navigator.geolocation.getCurrentPosition(onLocationSuccess);
+      }
+    }
+  
+    function initMap() {
+      const BOSNIA_BOUNDS = {
+        north: 45.7,
+        south: 41.69,
+        west: 15.55,
+        east: 20.77
+      };
+      const SARAJEVO_COORDINATES = {
+        lat: 43.85,
+        lng: 18.41
+      };
+  
+      const mapElement = document.getElementById("map");
+      const restrictMapPanningToBosniaOnly = {
+        latLngBounds: BOSNIA_BOUNDS,
+        strictBounds: true
+      };
+      const initialMapParams = {
+        center: SARAJEVO_COORDINATES,
+        zoom: 12,
+        restriction: restrictMapPanningToBosniaOnly,
+        mapTypeControl: false,
+        panControl: false,
+        zoomControl: true,
+        streetViewControl: false
+      };
+      map = new google.maps.Map(mapElement, initialMapParams);
+  
+      const inputElement = document.getElementById("autocompleteInput");
+      const restrictAutocompleteResultsToBosniaOnly = { country: "ba" };
+      const initialAutocompleteParams = {
+        types: ["geocode"],
+        componentRestrictions: restrictAutocompleteResultsToBosniaOnly,
+        fields: ["geometry", "types", "address_components"]
+      };
+  
+      autocomplete = new google.maps.places.Autocomplete(
+        inputElement,
+        initialAutocompleteParams
+      );
+      autocomplete.bindTo("bounds", map);
+      autocomplete.addListener("place_changed", onPlaceChanged);
+      pacSelectFirst(inputElement);
+      addLocateMeButton(map);
+      //Add event listener to position marker on map
+      google.maps.event.addListener(map, 'click', positionMarker);
+      
+    }
+
+    function positionMarker(event) {
+        let clickedLocation = event.latLng;
+        if(marker === false){
+            marker = new google.maps.Marker({
+                position: clickedLocation,
+                map: map,
+                draggable: true 
+            });
+            google.maps.event.addListener(marker, 'dragend', function(event){
+                markerLocation();
+            });
+        } else{
+            marker.setPosition(clickedLocation);
+        }
+    }
+
+    function addLocateMeButton(map) {
+      var parent = document.createElement("div");
+      parent.className = "locate-me-container";
+  
+      var a = document.createElement("a");
+      a.id = "locateMe";
+      a.className = "btn-floating";
+  
+      var i = document.createElement("i");
+      i.innerText = "gps_fixed";
+      i.className = "material-icons right";
+  
+      a.appendChild(i);
+      a.addEventListener("click", locateMe);
+      parent.appendChild(a);
+  
+      map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent);
+    }
+  
+    function onPlaceChanged() {
+      const place = autocomplete.getPlace();
+      if (place.geometry) {
+        map.fitBounds(place.geometry.viewport);
+        map.setZoom(map.getZoom() + 1);
+        $("#locationInputData").val(JSON.stringify(place));
+      }
+    }
+  
+    function pacSelectFirst(input) {
+      // store the original event binding function
+      const _addEventListener = input.addEventListener
+        ? input.addEventListener
+        : input.attachEvent;
+  
+      function addEventListenerWrapper(type, listener) {
+        // Simulate a 'down arrow' keypress on hitting 'return' when no pac suggestion is selected,
+        // and then trigger the original listener.
+        if (type == "keydown") {
+          const originalListener = listener;
+          listener = function(event) {
+            const suggestionSelected = $(".pac-item-selected").length > 0;
+            if (event.key == "Enter" && !suggestionSelected) {
+              const simulatedDownArrow = $.Event("keydown", {
+                keyCode: 40,
+                which: 40
+              });
+              originalListener.apply(input, [simulatedDownArrow]);
+            }
+  
+            originalListener.apply(input, [event]);
+          };
+        }
+  
+        _addEventListener.apply(input, [type, listener]);
+      }
+  
+      input.addEventListener = addEventListenerWrapper;
+      input.attachEvent = addEventListenerWrapper;
+    }
+      
+  
+  </script>
+  <script
+    src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
+    async
+    defer
+  ></script>
+  

app/views/publishPhotos.ejs

@@ -0,0 +1,12 @@
+<br>
+
+<div action="/photos-upload" class="dropzone" id="photos-upload">
+      <div class="fallback">
+        <input name="file" type="file" multiple />
+      </div>
+      
+    </div>
+
+<br>
+
+<input type="hidden" name="imageUrls" id="imageUrls" value="">

app/views/publishRealEstate.ejs

@@ -0,0 +1,290 @@
+<br>
+<form id="publishForm" method="POST" novalidate >
+  <div class="row">
+    <div class="col s12">
+      <ul class="tabs">
+        <li class="tab col s3"><a href="#publishBasicData">Osnovni podaci</a></li>
+        <li class="tab col s3"><a href="#publishAdditionalData">Dodatni podaci</a></li>
+        <li class="tab col s2"><a href="#publishLocation">Lokacija</a></li>
+        <li class="tab col s2"><a href="#publishPhotos">Fotografije</a></li>
+        <li class="tab col s2"><a href="#publishEnd">Kraj</a></li>
+
+    </ul>
+    </div>
+    <div id="publishBasicData" class="col s12">
+      <%- include("./publishBasicData.ejs") %>
+    </div>
+    <div id="publishAdditionalData" class="col s12">
+      <%- include("./publishAdditionalData.ejs") %>
+    </div>
+    <div id="publishLocation" class="col s12">
+     <%- include("./publishLocation.ejs") %>
+    </div>
+    <div id="publishPhotos" class="col s12">
+      <%- include("./publishPhotos.ejs") %>
+     </div>
+    <div id="publishEnd" class="col s12">
+     <%- include("./publishEnd.ejs") %>
+    </div>
+  </div>
+
+  
+</form>
+
+<script>
+  function uuidv4() {
+      return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+        const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
+        return v.toString(16);
+      });
+    }
+
+    function getFileName(fileName) {
+        const encodedFileName = (uuidv4() + fileName).replace(/\s+/g, '');
+        return encodedFileName;
+    }
+
+    function upload() {
+      var file = $('#selector')[0].files[0];
+      uploadFile(file)
+    }
+
+
+    async function generateSignedURL(file) {
+      const fileName = getFileName(file.name);
+      const response = await fetch('/generateSignedURL?filename=' + fileName);
+      if (!response.ok) {
+        throw new Error('Network response for fetch was not ok.');
+      }
+      let signedUrl = await response.text();
+      signedUrl = signedUrl.replace(/\"/g, "")
+      await uploadFile(file, fileName, signedUrl);
+      return fileName;
+    }
+
+       function uploadFile(file, fileName, url) {
+    return  fetch(url, {
+        method: 'PUT',
+        headers: new Headers({'content-type': 'image/*'}),
+        mode: 'cors',
+        body: file
+      })
+        .then(response => response.text())
+        .then (response => {
+          return response;
+        }
+        )
+        .catch(error => $("#status").html(error)
+        )
+        .then(response => {
+          $("#imageUrls").val($("#imageUrls").val()+ fileName+"|");
+        });
+
+    }
+
+  $(document).ready(function(){
+    $('.tabs').tabs();
+
+    // Manual dropzone init
+    const dropzoneOptions = {
+					url: "/photos-upload", //can be a function that returns url ?
+          autoProcessQueue:false, //not to upload files automaticly
+          method: "put", //or post
+          parallelUploads: 1,
+          uploadMultiple: false, 
+					addRemoveLinks: true,
+					maxFilesize: 2, //MB,
+          resizeWidth: 600, 
+          maxFiles: 10, 
+          acceptedFiles: "image/*",
+        	dictDefaultMessage: `<span class="text-center">
+                              <h3>Prevuci fotografije ili klikni za dodavanje!</h3>
+                              (Maksimalno 10 fotografija.)
+                              </span>`,
+					dictResponseError: 'Error uploading file!',
+          dictRemoveFile: 'Izbriši ',
+          dictFileTooBig: 'Fajl je prevelik!',
+          dictInvalidFileType: 'Iabrani fajl nije fotografija!',
+          dictMaxFilesExceeded: 'Dostigli ste maksimalan broj fotografija!'
+				};
+      var photosUploader = new Dropzone('#photos-upload', dropzoneOptions);
+
+
+    //VALIDATION - WiP
+    //Helper validation functions
+    const isValidEmail = $email => {
+      const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+      return $email && $email.length < 250 && simpleEmailRegex.test($email);
+      };
+
+    const isPresent = $input => {
+      return $input && $input!=="" && $input != null;
+      };
+
+    const isNumber = $input => {
+      const simpleNumberRegex = /([0-9]+[.|,][0-9])|([0-9][.|,][0-9]+)|([0-9]+)/;
+      return $input && $input.length <250 && simpleNumberRegex.test($input) && !isNaN($input);
+      };
+
+    const isInteger = $input => {
+      const simpleIntegerRegex = /^([+-]?[1-9]\d*|0)$/;
+      return $input && $input.length <250 && simpleIntegerRegex.test($input);
+      };
+
+    const form = document.querySelector("#publishForm");
+
+    function showErrors(form, errors) {
+    // We loop through all the inputs and show the errors for that input
+    //_.each(form.querySelectorAll("input[name], select[name]"), function(input) {
+      // Since the errors can be null if no errors were found we need to handle
+      // that
+      showErrorsForInput(input, errors && errors[input.name]);
+   // });
+  }
+
+  // Shows the errors for a specific input
+  function showErrorsForInput(input, errors) {
+    // This is the root of the input
+    var formGroup = closestParent(input.parentNode, "form-group"),
+      // Find where the error messages will be insert into
+      messages = formGroup.querySelector(".messages");
+    // First we remove any old messages and resets the classes
+    resetFormGroup(formGroup);
+    // If we have errors
+    if (errors) {
+      // we first mark the group has having errors
+      formGroup.classList.add("has-error");
+      // then we append all the errors
+      $.each(errors, function(error) {
+        addError(messages, errors[error]);
+      });
+
+    } else {
+      // otherwise we simply mark it as success
+      formGroup.classList.add("has-success");
+    }
+  }
+
+  // Recusively finds the closest parent that has the specified class
+  function closestParent(child, className) {
+    if (!child || child == document) {
+      return null;
+    }
+    if (child.classList.contains(className)) {
+      return child;
+    } else {
+      return closestParent(child.parentNode, className);
+    }
+  }
+
+  function resetFormGroup(formGroup) {
+    formGroup.classList.remove("has-error");
+    formGroup.classList.remove("has-success");
+    $.each(formGroup.querySelectorAll(".help-block.error"), function(el) {
+      el.parentNode.removeChild(el);
+    });
+  }
+
+  // Adds the specified error with the following markup
+  // <p class="help-block error">[message]</p>
+  function addError(messages, error) {
+    var block = document.createElement("p");
+    block.classList.add("help-block");
+    block.classList.add("error");
+    block.innerText = error;
+    messages.appendChild(block);
+  }
+
+  const validate = (input) => {
+
+    let valid=true;
+    let errorMsg =[];
+    let constraint = input.constraint[0];
+
+    switch (constraint) {
+      case "required":
+      valid = isPresent ($(`#${input.dbField}`).val());
+      errorMsg = ["Ovo je obavezno polje."];
+      break;
+      case "numerical":
+      valid = isNumber ($(`#${input.dbField}`).val());
+      errorMsg = ["Unesite brojcanu vrijednost."];
+      break;
+      case "integer":
+      valid = isInteger ($(`#${input.dbField}`).val());
+      errorMsg = ["Unesite cijeli broj."];
+      break;
+      default :
+      valid = true;
+     }
+    if (!valid) {
+    const inputField = document.querySelector(`#${input.dbField}`);
+    showErrorsForInput( inputField, errorMsg);
+    return false;
+    } else {
+    return true;
+    }
+
+}
+
+    $("#submit").click( async function (e) {
+      e.preventDefault();
+
+      if (marker) {
+        const currentLocation = marker.getPosition();
+
+        $("#lat").val(currentLocation.lat());
+
+        $("#lng").val(currentLocation.lng());
+
+        $("#locationInput").val(
+          document.getElementById("autocompleteInput").value
+          );
+      } else {
+        $("#lat").val(0);
+
+        $("#lng").val(0);
+      }
+      
+       //Tag for checking of error presence 
+       let hasErrors = false;
+      //Check if email is valid
+       const validEmail = isValidEmail($("#email").val());
+       //Show messeges for invalid email is present
+      if (!validEmail) {
+        const errorMsgs = ["Unesite validan email."];
+        const email = document.querySelector("#email");
+        showErrorsForInput( email, errorMsgs) 
+        hasErrors = true;
+      };
+      //Check if other input fields are valid - vratiti se na ovo!!
+      //const basicInputInputs= document.getElementById("basic-inputs").getElementsByTagName("input");
+      
+      //alert(JSON.stringify(""));
+      /*
+      $.each(basicInputInputs, function (input) {
+        alert(input);
+        validate (input);
+      })
+      for (const input of  basicInputInputs ) {
+        alert(input.getAttribute(name));
+
+        validate (input);
+      } */
+      const addedFiles = photosUploader.files.filter(file => file.status!=="error");
+      const asyncUpload =[];
+
+        addedFiles.forEach( file => {
+          asyncUpload.push(generateSignedURL(file)); 
+        })
+     
+        if (!hasErrors) {
+            await Promise.all(asyncUpload);
+            $("#publishForm").submit();
+
+      };
+  
+    });
+  });
+ 
+</script>

app/views/publishSuccess.ejs

@@ -0,0 +1,11 @@
+<br>
+<div class="row center-align">
+    <p>Vaš oglas je spašen u Kivi bazu.</p>
+    <br>
+    <div class="row center-align">
+        <img src="../assets/images/logo.svg" alt="kivi logo" width="160">
+    </div>
+    <br>
+    <p>Poslali smo potvrdni email sa detaljima oglasa na Vašu email adresu.</p>
+    <a href="/" class="">Nova pretraga</a>
+</div>

app/views/realEstateType.ejs

@@ -9,7 +9,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
+                    <span class="label"><%= labelAdType[0] %></span>
                 </label>
 
                 <label>
@@ -17,7 +17,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
+                    <span class="label"><%= labelAdType[1] %></span>
                 </label>
 		    </span>
         </div>

app/views/welcome.ejs

@@ -18,7 +18,20 @@
     </div>
     <input type="hidden" id="adType" name="adType">
 </form>
-
+<div class="row center-align">
+    <div>Objavite svoj oglas.</div>
+</div>
+<form method="POST" name="welcomePublishForm">
+    <div class="row center-align">
+        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
+            <a href="#" onclick="publishSaleClick()" class="welcome-center-button btn">Prodaj</a>
+        </div>
+        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
+            <a href="#" onclick="publishRentClick()" class="welcome-center-button btn">Iznajmi</a>
+        </div>
+    </div>
+    <input type="hidden" id="publishAdType" name="publishAdType">
+</form>
 <script>
     function saleClick(){
         $("#adType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
@@ -29,4 +42,13 @@
         $("#adType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
         document.welcomeForm.submit();
     }
+    function publishSaleClick(){
+        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
+        document.welcomePublishForm.submit();
+    }
+
+    function publishRentClick(){
+        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
+        document.welcomePublishForm.submit();
+    }
 </script>

development.env

@@ -8,9 +8,8 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-ENVIRONMENT=Variable to denote development, staging and production
+SETTINGS=Variable to denote development, staging and production
 
-USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
@@ -22,10 +21,8 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
-#=============== SCRAPER API SUPORT =============#
-USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api
-SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
+#=============== GOOGLE STORAGE =============#
+GOOGLE_APPLICATION_CREDENTIALS="Path to json key file"
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
@@ -72,8 +69,7 @@ AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==SALJIC NEKRETNINE==
-SALJIC_MAX_PAGES=Restrict crawler to this number of pages
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

help.js

@@ -0,0 +1,237 @@
+(function() {
+  // Before using it we must add the parse and format functions
+  // Here is a sample implementation using moment.js
+  validate.extend(validate.validators.datetime, {
+    // The value is guaranteed not to be null or undefined but otherwise it
+    // could be anything.
+    parse: function(value, options) {
+      return +moment.utc(value);
+    },
+    // Input is a unix timestamp
+    format: function(value, options) {
+      var format = options.dateOnly ? "YYYY-MM-DD" : "YYYY-MM-DD hh:mm:ss";
+      return moment.utc(value).format(format);
+    }
+  });
+
+  // These are the constraints used to validate the form
+  var constraints = {
+    email: {
+      // Email is required
+      presence: true,
+      // and must be an email (duh)
+      email: true
+    },
+    password: {
+      // Password is also required
+      presence: true,
+      // And must be at least 5 characters long
+      length: {
+        minimum: 5
+      }
+    },
+    "confirm-password": {
+      // You need to confirm your password
+      presence: true,
+      // and it needs to be equal to the other password
+      equality: {
+        attribute: "password",
+        message: "^The passwords does not match"
+      }
+    },
+    username: {
+      // You need to pick a username too
+      presence: true,
+      // And it must be between 3 and 20 characters long
+      length: {
+        minimum: 3,
+        maximum: 20
+      },
+      format: {
+        // We don't allow anything that a-z and 0-9
+        pattern: "[a-z0-9]+",
+        // but we don't care if the username is uppercase or lowercase
+        flags: "i",
+        message: "can only contain a-z and 0-9"
+      }
+    },
+    birthdate: {
+      // The user needs to give a birthday
+      presence: true,
+      // and must be born at least 18 years ago
+      date: {
+        latest: moment().subtract(18, "years"),
+        message: "^You must be at least 18 years old to use this service"
+      }
+    },
+    country: {
+      // You also need to input where you live
+      presence: true,
+      // And we restrict the countries supported to Sweden
+      inclusion: {
+        within: ["SE"],
+        // The ^ prevents the field name from being prepended to the error
+        message: "^Sorry, this service is for Sweden only"
+      }
+    },
+    zip: {
+      // Zip is optional but if specified it must be a 5 digit long number
+      format: {
+        pattern: "\\d{5}"
+      }
+    },
+    "number-of-children": {
+      presence: true,
+      // Number of children has to be an integer >= 0
+      numericality: {
+        onlyInteger: true,
+        greaterThanOrEqualTo: 0
+      }
+    }
+  };
+
+  // Hook up the form so we can prevent it from being posted
+  var form = document.querySelector("form#main");
+  form.addEventListener("submit", function(ev) {
+    ev.preventDefault();
+    handleFormSubmit(form);
+  });
+
+  // Hook up the inputs to validate on the fly
+  var inputs = document.querySelectorAll("input, textarea, select");
+  for (var i = 0; i < inputs.length; ++i) {
+    inputs.item(i).addEventListener("change", function(ev) {
+      var errors = validate(form, constraints) || {};
+      showErrorsForInput(this, errors[this.name]);
+    });
+  }
+
+  function handleFormSubmit(form, input) {
+    // validate the form against the constraints
+    var errors = validate(form, constraints);
+    // then we update the form to reflect the results
+    showErrors(form, errors || {});
+    if (!errors) {
+      showSuccess();
+    }
+  }
+
+  // Updates the inputs with the validation errors
+  function showErrors(form, errors) {
+    // We loop through all the inputs and show the errors for that input
+    _.each(form.querySelectorAll("input[name], select[name]"), function(input) {
+      // Since the errors can be null if no errors were found we need to handle
+      // that
+      showErrorsForInput(input, errors && errors[input.name]);
+    });
+  }
+
+  // Shows the errors for a specific input
+  function showErrorsForInput(input, errors) {
+    // This is the root of the input
+    var formGroup = closestParent(input.parentNode, "form-group"),
+      // Find where the error messages will be insert into
+      messages = formGroup.querySelector(".messages");
+    // First we remove any old messages and resets the classes
+    resetFormGroup(formGroup);
+    // If we have errors
+    if (errors) {
+      // we first mark the group has having errors
+      formGroup.classList.add("has-error");
+      // then we append all the errors
+      _.each(errors, function(error) {
+        addError(messages, error);
+      });
+    } else {
+      // otherwise we simply mark it as success
+      formGroup.classList.add("has-success");
+    }
+  }
+
+  // Recusively finds the closest parent that has the specified class
+  function closestParent(child, className) {
+    if (!child || child == document) {
+      return null;
+    }
+    if (child.classList.contains(className)) {
+      return child;
+    } else {
+      return closestParent(child.parentNode, className);
+    }
+  }
+
+  function resetFormGroup(formGroup) {
+    // Remove the success and error classes
+    formGroup.classList.remove("has-error");
+    formGroup.classList.remove("has-success");
+    // and remove any old messages
+    _.each(formGroup.querySelectorAll(".help-block.error"), function(el) {
+      el.parentNode.removeChild(el);
+    });
+  }
+
+  // Adds the specified error with the following markup
+  // <p class="help-block error">[message]</p>
+  function addError(messages, error) {
+    var block = document.createElement("p");
+    block.classList.add("help-block");
+    block.classList.add("error");
+    block.innerText = error;
+    messages.appendChild(block);
+  }
+
+  function showSuccess() {
+    // We made it \:D/
+    alert("Success!");
+  }
+})();
+/////////////////////////////////////////////////
+const isPresent = $input => {
+  return $input && $input!=="" && $input != null;
+}
+
+const isNumber = $input => {
+  const simpleNumberRegex = /[+-]?(?:\d*[.,])?\d+/;
+  return $input && $input.length <250 && simpleNumberRegex.test($input);
+
+}
+
+const isInteger = $input => {
+  const simpleIntegerRegex = /^([+-]?[1-9]\d*|0)$/;
+  return $input && $input.length <250 && simpleIntegerRegex.test($input);
+
+}
+
+const validate = (input) => {
+
+  const valid;
+  const errorMsg;
+  const constraint = input.constraint[0];
+
+  switch (constraint) {
+    case "required":
+      valid = isPresent ($(`#${input.dbField}`).val());
+      errorMsg = ["Ovo je obavezno polje."];
+      break;
+    case "numerical":
+      valid = isNumber ($(`#${input.dbField}`).val());
+      errorMsg = ["Unesite brojcanu vrijednost."];
+      break;
+    case "integer":
+      valid = isInteger ($(`#${input.dbField}`).val());
+      errorMsg = ["Unesite cijeli broj."];
+
+      break;
+      default :
+      valid = true;
+    }
+    if (!valid) {
+      const inputField = document.querySelector(`#${input.dbField}`);
+    showErrorsForInput( inputField, errorMsg);
+    return false;
+  } else {
+    return true;
+  }
+
+  }
+

help2.js

@@ -0,0 +1,112 @@
+const validatorFunction = () => {
+  // These are the constraints used to validate the form --just email for now!
+  const constraints = {
+    email: {
+      email: {
+        message: "Proba"
+      },
+      // Email is required
+      presence: true,
+      // and must be an email (duh)
+      email: true
+    }
+  };
+
+  // Hook up the inputs to validate on the fly
+  const inputs = document.querySelectorAll("#email");
+  // inputs.on("change", ev => {
+  //     const errors = validate(form, constraints) || {};
+  //     showErrorsForInput(this, errors[this.name]);
+  //   });
+  // var inputs = document.querySelectorAll("input, textarea, select");
+  for (var i = 0; i < inputs.length; ++i) {
+    inputs.item(i).addEventListener("change", function(ev) {
+      var errors = validate(form, constraints) || {};
+      showErrorsForInput(this, errors[this.name]);
+    });
+  }
+
+  const handleFormSubmit = (form, input) => {
+    // validate the form against the constraints
+    const errors = validate(form, constraints);
+    //
+    console.log("handleFormSubmit error:", errors);
+    // then we update the form to reflect the results
+    showErrors(form, errors || {});
+    if (!errors) {
+      showSuccess();
+    }
+  };
+
+  // Updates the inputs with the validation errors
+  const showErrors = (form, errors) => {
+    // We loop through all the inputs and show the errors for that input
+    $.each(form.querySelectorAll("input[name], select[name]"), input => {
+      // Since the errors can be null if no errors were found we need to handle
+      // that
+      showErrorsForInput(input, errors && errors[input.name]);
+    });
+    //showErrorsForInput(email, errors && errors[email]);
+  };
+
+  // Shows the errors for a specific input
+  const showErrorsForInput = (input, errors) => {
+    // This is the root of the input
+    const formGroup = closestParent(input.parentNode, "form-group"),
+      // Find where the error messages will be insert into
+      messages = formGroup.querySelector(".messages");
+    // First we remove any old messages and resets the classes
+    resetFormGroup(formGroup);
+    // If we have errors
+    if (errors) {
+      //
+      console.log("errors:", errors);
+      // we first mark the group has having errors
+      formGroup.classList.add("has-error");
+      // then we append all the errors
+      $.each(errors, error => {
+        addError(messages, errors[error]);
+      });
+    } else {
+      // otherwise we simply mark it as success
+      formGroup.classList.add("has-success");
+    }
+  };
+
+  // Recusively finds the closest parent that has the specified class
+  const closestParent = (child, className) => {
+    if (!child || child == document) {
+      return null;
+    }
+    if (child.classList.contains(className)) {
+      return child;
+    } else {
+      return closestParent(child.parentNode, className);
+    }
+  };
+
+  const resetFormGroup = formGroup => {
+    // Remove the success and error classes
+    formGroup.classList.remove("has-error");
+    formGroup.classList.remove("has-success");
+    // and remove any old messages
+    $.each(formGroup.querySelectorAll(".help-block.error"), el => {
+      el.parentNode.removeChild(el);
+    });
+  };
+
+  // Adds the specified error with the following markup
+  // <p class="help-block error">[message]</p>
+  const addError = (messages, error) => {
+    const block = document.createElement("p");
+    block.classList.add("help-block");
+    block.classList.add("error");
+    block.innerText = error;
+    messages.appendChild(block);
+  };
+
+  const showSuccess = () => {
+    // We made it \:D/
+    alert("Success!");
+  };
+};

index.js

@@ -4,7 +4,10 @@ const bodyParser = require("body-parser");
 const layout = require("express-layout");
 const compression = require("compression");
 const forceSSL = require("./app/helpers/forceSSL");
-const { logDebug } = require("./app/helpers/log");
+
+const { Storage } = require("@google-cloud/storage");
+const validate = require("validate.js");
+const cors = require("cors");
 
 const {
   APP_PORT,
@@ -20,6 +23,8 @@ const {
 
 const app = express();
 
+app.use(cors());
+
 app.use(forceSSL());
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({ extended: true }));
@@ -39,17 +44,11 @@ app.listen(APP_PORT, () =>
 
 let crawlerRunning = STOP_CRAWLER;
 const crawl = () => {
-  logDebug("Crawl start. crawlerRunning: ", crawlerRunning);
   if (!crawlerRunning) {
     crawlerRunning = true;
     crawlAll().then(newRealEstates => {
-      logDebug("crawlAll done, new real estate len: ", newRealEstates.length);
-      notifyForNewRealEstates(newRealEstates);
-    }).catch(e => {
-      console.error('Error happened: ', e);
-    }).finally(()=> {
       crawlerRunning = false;
-      logDebug('Finally done!');
+      notifyForNewRealEstates(newRealEstates);
     });
   }
 };
@@ -57,3 +56,36 @@ const crawl = () => {
 setInterval(crawl, CRAWLER_INTERVAL * 1000);
 
 setInterval(checkUpNotify, 1000 * 60 * 60 * 24);
+
+//Google storage req
+const PROJECT_ID = "marketalarm";
+const KEY_FILENAME = ""; //relative path
+const BUCKET_NAME = "marketalarm-photos";
+const storage = new Storage();
+
+const bucket = storage.bucket(BUCKET_NAME);
+
+app.get("/generateSignedURL", (req, res) => {
+  async function generateSignedUrl() {
+    // console.log("Started server function!");
+
+    const options = {
+      //Tried to define Google ID and private key while debugging
+      version: "v2", //tried v4 also
+      action: "write",
+      contentType: "image/*", //tried without and with specific image/png ex.
+      expires: Date.now() + 86400000
+    };
+    const filename = req.query.filename;
+
+    // console.log("Filename: ", filename);
+    // console.log("Bucket name:", bucket.name);
+
+    const [url] = await bucket.file(filename).getSignedUrl(options);
+
+    //console.log(`The signed url is ${url}.`);
+
+    res.status(200).send(url);
+  }
+  generateSignedUrl().catch(console.error);
+});

marketalarm-cors.json

@@ -0,0 +1,12 @@
+[
+  {
+    "origin": ["*"],
+    "responseHeader": [
+      "Content-Type",
+      "Access-Control-Allow-Origin",
+      "x-goog-resumable"
+    ],
+    "method": ["GET", "HEAD", "DELETE", "POST", "PUT", "OPTIONS"],
+    "maxAgeSeconds": 3600
+  }
+]

marketalarm-d4b71394407e.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "marketalarm",
+  "private_key_id": "d4b71394407eb3dba9e431851dab60b198d6985d",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDTE6dkFr0bzDXg\n7ghMxkzq8cajqqqes9JZVqsXh+b/kFJYmEImFUILJJZdI080KM2sEYsIapBCxhMP\nFH017f/gfH3jnRbp3c70hghNh8noSTsq7kPA4l25o8GQnJ6AS+nhy8umPjb4KzX9\nkmC6OOD4P8mAmGqhoUv4s2jld1cXNur6NJjCpjEd2cH3SUbI71oA3V/4W8aK4dvS\n660kLY0PRt7mCiITe0hbTUBZY48W2ijZ7wM2r0HUtPG9XEeGMGmNsC+qD2oWxUU3\nvnm7l1fEIUvLYF4GrLRDJDSkpChBXNcWhoGV2AOvuTc+yghU2+lJWqrKcpLlI23E\nlVJjt9UhAgMBAAECggEABatr8sxq+SQOf9hSIe3Me9Kc1nunrC42scFHRKBNxahJ\ndXw5B9FQPh738Cqhk0xEz6hlrln1Agj6HhRIz8U0r9R+z4TRRr6kfnWmBZAMShu0\nC4JW448abpAYx8CQ/CvRmq2GlF+/M+QBeqpLS8gPzyaKTB/5IBaKG8Bn0fXXQZ2e\n7RaTpGx62jq79omPwiKz0PMVBGZrzPu8Z4tW47muV51osdKSNVgsXb4gCZl28zN2\n6zzY1ZK7u89MesY8joILMHm8cw0oyv9o+RVGEa1I1nq2q1A8ftZny4p7kUA/ITZX\nEZ6SCOP87z9HeVCr8lzexcovD8uZCOTYpcfotlSjGQKBgQD1VVGU2bzExiV0XQGc\n8n6m4TR6Y9zwXBiQPe0rXPZhvsj8QMTXk+L0ejCo4m7NF1dEyH6u+qX6wjNL1Hm9\nN/ZuDFqYtd9w6cQ8CtDZ6QZIE60k6tLQhMNRNMvHdMfedq+VOz3LX6TdyTnv8dP3\nbEsD8wIfFd6t5wNgeZkbKsNxBwKBgQDcQQsUppjglGpUoz7lGHKbFcKMPpIj9fMY\nfze1DXeTAtHGxGm2F10WZvxOEs4DCOUllBlarL5xDAJIJHk/NYlgnI6MJXMLro6d\nsb4iNTUuJKdqAijyOaZQUADJpdYKGwu5y66PUOuojWFV9kiamquXduJ9jzOa1vr1\nSJPUy2YGlwKBgQDNJrpgwa8z0QozAy89Ih68x+fNTMLNkAXOYKp6L3OsixCguDyi\nlP0dOSyFnUvQXutQDmS5R8oSJeElURk4HJsKrXP47WVak3DQUK8S+eSR0zpfe6os\nSkjWGFMriEE2i4MKRI7JCULhX8r+FfgNl9YnCEfG3M/oFhzhyO06JYlncwKBgF6n\nBSAGyEQbA+cDkI/bhcToAQdMDHmvxJyOb147P1vKJmSJG/TI7ZQnBd53blkXhYI0\ntwCko+LpCkH+iqyDUVpXbVsE7P/kMB3MuKzyuLvvvJJuAzK1W6e/+daukeEd5lge\nFBI68EsrFt1eTa1DMuKQkJzs4Xx1TrwCSKV2E45ZAoGBAIZkXyAOhwqCxwDF7B69\nt/7CWs0gPGqp6lFO7fgt7jPmcmSEr/xgUbBDFwd7D49jpXVgCEtr1Bd6MItlu/Ns\nXgXyOa5LPQmglF7UtnvuQLASBy5X6boKaf3sz7I5eho1kXczPGQUHfR5e0DaTND3\nTi2NLIAUci8T7hc8mONdeEHD\n-----END PRIVATE KEY-----\n",
+  "client_email": "marketalarm-photos-service-acc@marketalarm.iam.gserviceaccount.com",
+  "client_id": "115644068453290488813",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/marketalarm-photos-service-acc%40marketalarm.iam.gserviceaccount.com"
+}

package-lock.json

@@ -12,6 +12,92 @@
         "request": "2.x.x"
       }
     },
+    "@google-cloud/common": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-2.4.0.tgz",
+      "integrity": "sha512-zWFjBS35eI9leAHhjfeOYlK5Plcuj/77EzstnrJIZbKgF/nkqjcQuGiMCpzCwOfPyUbz8ZaEOYgbHa759AKbjg==",
+      "requires": {
+        "@google-cloud/projectify": "^1.0.0",
+        "@google-cloud/promisify": "^1.0.0",
+        "arrify": "^2.0.0",
+        "duplexify": "^3.6.0",
+        "ent": "^2.2.0",
+        "extend": "^3.0.2",
+        "google-auth-library": "^5.5.0",
+        "retry-request": "^4.0.0",
+        "teeny-request": "^6.0.0"
+      }
+    },
+    "@google-cloud/paginator": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-2.0.3.tgz",
+      "integrity": "sha512-kp/pkb2p/p0d8/SKUu4mOq8+HGwF8NPzHWkj+VKrIPQPyMRw8deZtrO/OcSiy9C/7bpfU5Txah5ltUNfPkgEXg==",
+      "requires": {
+        "arrify": "^2.0.0",
+        "extend": "^3.0.2"
+      }
+    },
+    "@google-cloud/projectify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-1.0.4.tgz",
+      "integrity": "sha512-ZdzQUN02eRsmTKfBj9FDL0KNDIFNjBn/d6tHQmA/+FImH5DO6ZV8E7FzxMgAUiVAUq41RFAkb25p1oHOZ8psfg=="
+    },
+    "@google-cloud/promisify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-1.0.4.tgz",
+      "integrity": "sha512-VccZDcOql77obTnFh0TbNED/6ZbbmHDf8UMNnzO1d5g9V0Htfm4k5cllY8P1tJsRKC3zWYGRLaViiupcgVjBoQ=="
+    },
+    "@google-cloud/storage": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-4.5.0.tgz",
+      "integrity": "sha512-ZLFcR6CiP1AnYBA9eTtASF9Dy3wjYmGx+HZiy/LsIPN41wyBTn9yAjIOxRHiteqzX3uQzZ+VJNCB/DmTU33CeQ==",
+      "requires": {
+        "@google-cloud/common": "^2.1.1",
+        "@google-cloud/paginator": "^2.0.0",
+        "@google-cloud/promisify": "^1.0.0",
+        "arrify": "^2.0.0",
+        "compressible": "^2.0.12",
+        "concat-stream": "^2.0.0",
+        "date-and-time": "^0.12.0",
+        "duplexify": "^3.5.0",
+        "extend": "^3.0.2",
+        "gaxios": "^2.0.1",
+        "gcs-resumable-upload": "^2.2.4",
+        "hash-stream-validation": "^0.2.2",
+        "mime": "^2.2.0",
+        "mime-types": "^2.0.8",
+        "onetime": "^5.1.0",
+        "p-limit": "^2.2.0",
+        "pumpify": "^2.0.0",
+        "readable-stream": "^3.4.0",
+        "snakeize": "^0.1.0",
+        "stream-events": "^1.0.1",
+        "through2": "^3.0.0",
+        "xdg-basedir": "^4.0.0"
+      },
+      "dependencies": {
+        "mime": {
+          "version": "2.4.4",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
+          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
+        },
+        "readable-stream": {
+          "version": "3.6.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
+          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        },
+        "xdg-basedir": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz",
+          "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q=="
+        }
+      }
+    },
     "@sendgrid/client": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/@sendgrid/client/-/client-6.3.0.tgz",
@@ -40,31 +126,10 @@
         "@sendgrid/helpers": "^6.3.0"
       }
     },
-    "@sozialhelden/fetch-cache": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@sozialhelden/fetch-cache/-/fetch-cache-2.0.1.tgz",
-      "integrity": "sha512-vMlsdT5JQCGjx1fcFxmMNh7ZKppjjsfUAeZEhhNwhEL7GaqbZXsD1OXEyx2IcRa25ZuZtvJSV6Q3rE77VRdLvg==",
-      "requires": {
-        "@sozialhelden/hamster-cache": "^1.0.0"
-      }
-    },
-    "@sozialhelden/hamster-cache": {
+    "@tootallnate/once": {
       "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@sozialhelden/hamster-cache/-/hamster-cache-1.0.0.tgz",
-      "integrity": "sha512-/TEGA8mdMawZp4Yq/GrkL+72YL5EGuSeVXC3pKW12YY1t3C+zCN/HZ0HRp4zWF/e67svXcxuz/B0AEQxEdvi7A=="
-    },
-    "@supercharge/goodies": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
-      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
-    },
-    "@supercharge/promise-pool": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
-      "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
-      "requires": {
-        "@supercharge/goodies": "~1.4.0"
-      }
+      "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.0.0.tgz",
+      "integrity": "sha512-KYyTT/T6ALPkIRd2Ge080X/BsXvy9O0hcWTtMWkPvwAwF99+vn6Dv4GzrFT/Nn1LePr+FFDbRXXlqmsy9lw2zA=="
     },
     "@types/caseless": {
       "version": "0.12.2",
@@ -122,6 +187,29 @@
         "negotiator": "0.6.1"
       }
     },
+    "agent-base": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz",
+      "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==",
+      "requires": {
+        "debug": "4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+        }
+      }
+    },
     "ajv": {
       "version": "6.10.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
@@ -210,6 +298,11 @@
       "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
       "dev": true
     },
+    "arrify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz",
+      "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug=="
+    },
     "asn1": {
       "version": "0.2.4",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -354,6 +447,11 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "bignumber.js": {
+      "version": "7.2.1",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
+      "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
+    },
     "binary-extensions": {
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
@@ -458,6 +556,16 @@
         "isarray": "^1.0.0"
       }
     },
+    "buffer-equal-constant-time": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
+      "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
+    },
+    "buffer-from": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+    },
     "buffer-writer": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
@@ -705,6 +813,17 @@
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
     },
+    "concat-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+      "requires": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.0.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "config-chain": {
       "version": "1.1.12",
       "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz",
@@ -764,6 +883,15 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "cors": {
+      "version": "2.8.5",
+      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
+      "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
+      "requires": {
+        "object-assign": "^4",
+        "vary": "^1"
+      }
+    },
     "create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -833,6 +961,11 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "date-and-time": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.12.0.tgz",
+      "integrity": "sha512-n2RJIAp93AucgF/U/Rz5WRS2Hjg5Z+QxscaaMCi6pVZT1JpJKRH+C08vyH/lRR1kxNXnPxgo3lWfd+jCb/UcuQ=="
+    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -983,6 +1116,41 @@
       "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
       "dev": true
     },
+    "duplexify": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
+      "requires": {
+        "end-of-stream": "^1.0.0",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0",
+        "stream-shift": "^1.0.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
     "ecc-jsbn": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -992,6 +1160,14 @@
         "safer-buffer": "^2.1.0"
       }
     },
+    "ecdsa-sig-formatter": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
+      "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
+      "requires": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
     "editorconfig": {
       "version": "0.15.3",
       "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
@@ -1031,6 +1207,11 @@
         "once": "^1.4.0"
       }
     },
+    "ent": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz",
+      "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0="
+    },
     "entities": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
@@ -1328,6 +1509,11 @@
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
       "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
     },
+    "fast-text-encoding": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.1.tgz",
+      "integrity": "sha512-x4FEgaz3zNRtJfLFqJmHWxkMDDvXVtaznj2V9jiP8ACUJrUgist4bP9FmDL2Vew2Y9mEQI/tG4GqabaitYp9CQ=="
+    },
     "fill-range": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
@@ -1986,6 +2172,117 @@
         }
       }
     },
+    "gaxios": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.3.2.tgz",
+      "integrity": "sha512-K/+py7UvKRDaEwEKlLiRKrFr+wjGjsMz5qH7Vs549QJS7cpSCOT/BbWL7pzqECflc46FcNPipjSfB+V1m8PAhw==",
+      "requires": {
+        "abort-controller": "^3.0.0",
+        "extend": "^3.0.2",
+        "https-proxy-agent": "^5.0.0",
+        "is-stream": "^2.0.0",
+        "node-fetch": "^2.3.0"
+      },
+      "dependencies": {
+        "is-stream": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
+          "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
+        }
+      }
+    },
+    "gcp-metadata": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.5.0.tgz",
+      "integrity": "sha512-ZQf+DLZ5aKcRpLzYUyBS3yo3N0JSa82lNDO8rj3nMSlovLcz2riKFBsYgDzeXcv75oo5eqB2lx+B14UvPoCRnA==",
+      "requires": {
+        "gaxios": "^2.1.0",
+        "json-bigint": "^0.3.0"
+      }
+    },
+    "gcs-resumable-upload": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-2.3.2.tgz",
+      "integrity": "sha512-OPS0iAmPCV+r7PziOIhyxmQOzsazFCy76yYDOS/Z80O/7cuny1KMfqDQa2T0jLaL8EreTU7EMZG5pUuqBKgzHA==",
+      "requires": {
+        "abort-controller": "^3.0.0",
+        "configstore": "^5.0.0",
+        "gaxios": "^2.0.0",
+        "google-auth-library": "^5.0.0",
+        "pumpify": "^2.0.0",
+        "stream-events": "^1.0.4"
+      },
+      "dependencies": {
+        "configstore": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
+          "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==",
+          "requires": {
+            "dot-prop": "^5.2.0",
+            "graceful-fs": "^4.1.2",
+            "make-dir": "^3.0.0",
+            "unique-string": "^2.0.0",
+            "write-file-atomic": "^3.0.0",
+            "xdg-basedir": "^4.0.0"
+          }
+        },
+        "crypto-random-string": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz",
+          "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA=="
+        },
+        "dot-prop": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
+          "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
+          "requires": {
+            "is-obj": "^2.0.0"
+          }
+        },
+        "is-obj": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
+          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
+        },
+        "make-dir": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.2.tgz",
+          "integrity": "sha512-rYKABKutXa6vXTXhoV18cBE7PaewPXHe/Bdq4v+ZLMhxbWApkFFplT0LcbMW+6BbjnQXzZ/sAvSE/JdguApG5w==",
+          "requires": {
+            "semver": "^6.0.0"
+          }
+        },
+        "semver": {
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
+        },
+        "unique-string": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz",
+          "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==",
+          "requires": {
+            "crypto-random-string": "^2.0.0"
+          }
+        },
+        "write-file-atomic": {
+          "version": "3.0.3",
+          "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz",
+          "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==",
+          "requires": {
+            "imurmurhash": "^0.1.4",
+            "is-typedarray": "^1.0.0",
+            "signal-exit": "^3.0.2",
+            "typedarray-to-buffer": "^3.1.5"
+          }
+        },
+        "xdg-basedir": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz",
+          "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q=="
+        }
+      }
+    },
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -2056,6 +2353,45 @@
         "ini": "^1.3.4"
       }
     },
+    "google-auth-library": {
+      "version": "5.10.1",
+      "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.10.1.tgz",
+      "integrity": "sha512-rOlaok5vlpV9rSiUu5EpR0vVpc+PhN62oF4RyX/6++DG1VsaulAFEMlDYBLjJDDPI6OcNOCGAKy9UVB/3NIDXg==",
+      "requires": {
+        "arrify": "^2.0.0",
+        "base64-js": "^1.3.0",
+        "ecdsa-sig-formatter": "^1.0.11",
+        "fast-text-encoding": "^1.0.0",
+        "gaxios": "^2.1.0",
+        "gcp-metadata": "^3.4.0",
+        "gtoken": "^4.1.0",
+        "jws": "^4.0.0",
+        "lru-cache": "^5.0.0"
+      },
+      "dependencies": {
+        "lru-cache": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+          "requires": {
+            "yallist": "^3.0.2"
+          }
+        },
+        "yallist": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+        }
+      }
+    },
+    "google-p12-pem": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz",
+      "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==",
+      "requires": {
+        "node-forge": "^0.9.0"
+      }
+    },
     "got": {
       "version": "6.7.1",
       "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -2088,6 +2424,24 @@
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz",
       "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA=="
     },
+    "gtoken": {
+      "version": "4.1.4",
+      "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz",
+      "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==",
+      "requires": {
+        "gaxios": "^2.1.0",
+        "google-p12-pem": "^2.0.0",
+        "jws": "^4.0.0",
+        "mime": "^2.2.0"
+      },
+      "dependencies": {
+        "mime": {
+          "version": "2.4.4",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
+          "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
+        }
+      }
+    },
     "har-schema": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
@@ -2139,6 +2493,47 @@
         }
       }
     },
+    "hash-stream-validation": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.2.tgz",
+      "integrity": "sha512-cMlva5CxWZOrlS/cY0C+9qAzesn5srhFA8IT1VPiHc9bWWBLkJfEUIZr7MWoi89oOOGmpg8ymchaOjiArsGu5A==",
+      "requires": {
+        "through2": "^2.0.0"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        },
+        "through2": {
+          "version": "2.0.5",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
+          "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==",
+          "requires": {
+            "readable-stream": "~2.3.6",
+            "xtend": "~4.0.1"
+          }
+        }
+      }
+    },
     "he": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
@@ -2186,6 +2581,31 @@
         "statuses": ">= 1.4.0 < 2"
       }
     },
+    "http-proxy-agent": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
+      "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==",
+      "requires": {
+        "@tootallnate/once": "1",
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+        }
+      }
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -2196,6 +2616,30 @@
         "sshpk": "^1.7.0"
       }
     },
+    "https-proxy-agent": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
+      "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
+      "requires": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+        }
+      }
+    },
     "iconv-lite": {
       "version": "0.4.23",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
@@ -2224,8 +2668,7 @@
     "imurmurhash": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
-      "dev": true
+      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o="
     },
     "inflection": {
       "version": "1.12.0",
@@ -2519,6 +2962,14 @@
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
       "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
     },
+    "json-bigint": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz",
+      "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=",
+      "requires": {
+        "bignumber.js": "^7.0.0"
+      }
+    },
     "json-schema": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
@@ -2553,6 +3004,25 @@
         "verror": "1.10.0"
       }
     },
+    "jwa": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
+      "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
+      "requires": {
+        "buffer-equal-constant-time": "1.0.1",
+        "ecdsa-sig-formatter": "1.0.11",
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "jws": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
+      "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
+      "requires": {
+        "jwa": "^2.0.0",
+        "safe-buffer": "^5.0.1"
+      }
+    },
     "kind-of": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
@@ -2840,6 +3310,11 @@
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
       "integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
     },
+    "node-forge": {
+      "version": "0.9.1",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz",
+      "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ=="
+    },
     "node-schedule": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-1.3.2.tgz",
@@ -2926,6 +3401,11 @@
       "resolved": "https://registry.npmjs.org/obj-extend/-/obj-extend-0.1.0.tgz",
       "integrity": "sha1-u0SKR3X7les0p4H5CLusLfI9u1s="
     },
+    "object-assign": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
+      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
+    },
     "object-copy": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
@@ -3001,6 +3481,14 @@
         "wrappy": "1"
       }
     },
+    "onetime": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
+      "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
+      "requires": {
+        "mimic-fn": "^2.1.0"
+      }
+    },
     "os-homedir": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
@@ -3257,8 +3745,7 @@
     "process-nextick-args": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
-      "dev": true
+      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
     },
     "proto-list": {
       "version": "1.2.4",
@@ -3299,6 +3786,29 @@
         "once": "^1.3.1"
       }
     },
+    "pumpify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz",
+      "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==",
+      "requires": {
+        "duplexify": "^4.1.1",
+        "inherits": "^2.0.3",
+        "pump": "^3.0.0"
+      },
+      "dependencies": {
+        "duplexify": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz",
+          "integrity": "sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA==",
+          "requires": {
+            "end-of-stream": "^1.4.1",
+            "inherits": "^2.0.3",
+            "readable-stream": "^3.1.1",
+            "stream-shift": "^1.0.0"
+          }
+        }
+      }
+    },
     "punycode": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@@ -3531,6 +4041,30 @@
         "any-promise": "^1.3.0"
       }
     },
+    "retry-request": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz",
+      "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==",
+      "requires": {
+        "debug": "^4.1.1",
+        "through2": "^3.0.1"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+        }
+      }
+    },
     "safe-buffer": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@@ -3723,6 +4257,11 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
     },
+    "snakeize": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz",
+      "integrity": "sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0="
+    },
     "snapdragon": {
       "version": "0.8.2",
       "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
@@ -3919,6 +4458,19 @@
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
       "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
     },
+    "stream-events": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
+      "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==",
+      "requires": {
+        "stubs": "^3.0.0"
+      }
+    },
+    "stream-shift": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
+      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
+    },
     "string-width": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
@@ -3965,6 +4517,11 @@
       "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
       "dev": true
     },
+    "stubs": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz",
+      "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls="
+    },
     "supports-color": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
@@ -3973,6 +4530,18 @@
         "has-flag": "^3.0.0"
       }
     },
+    "teeny-request": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.2.tgz",
+      "integrity": "sha512-B6fxA0fSnY/bul06NggdN1nywtr5U5Uvt96pHfTi8pi4MNe6++VUWcAAFBrcMeha94s+gULwA5WvagoSZ+AcYg==",
+      "requires": {
+        "http-proxy-agent": "^4.0.0",
+        "https-proxy-agent": "^5.0.0",
+        "node-fetch": "^2.2.0",
+        "stream-events": "^1.0.5",
+        "uuid": "^3.3.2"
+      }
+    },
     "term-size": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz",
@@ -4021,6 +4590,14 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
     },
+    "through2": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz",
+      "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==",
+      "requires": {
+        "readable-stream": "2 || 3"
+      }
+    },
     "timed-out": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",
@@ -4146,6 +4723,19 @@
         "mime-types": "~2.1.18"
       }
     },
+    "typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
+    },
+    "typedarray-to-buffer": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
+      "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==",
+      "requires": {
+        "is-typedarray": "^1.0.0"
+      }
+    },
     "umzug": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.2.0.tgz",
@@ -4348,6 +4938,11 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
       "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
     },
+    "validate.js": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz",
+      "integrity": "sha512-PnFM3xiZ+kYmLyTiMgTYmU7ZHkjBZz2/+F0DaALc/uUtVzdCt1wAosvYJ5hFQi/hz8O4zb52FQhHZRC+uVkJ+g=="
+    },
     "validator": {
       "version": "10.11.0",
       "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",

package.json

@@ -17,8 +17,7 @@
     "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js",
-    "test-rental-scraper": "cd test && node rentalScrapeTest.js",
-    "test-saljic-scraper": "cd test && node saljicScrapeTest.js"
+    "test-rental-scraper": "cd test && node rentalScrapeTest.js"
   },
   "repository": {
     "type": "git",
@@ -31,14 +30,13 @@
   },
   "dependencies": {
     "2checkout-node": "0.0.1",
+    "@google-cloud/storage": "^4.5.0",
     "@sendgrid/mail": "^6.3.1",
-    "@sozialhelden/fetch-cache": "^2.0.1",
-    "@supercharge/promise-pool": "^1.3.0",
-    "abort-controller": "^3.0.0",
     "aws-sdk": "^2.422.0",
     "bluebird": "^3.5.5",
     "cheerio": "^1.0.0-rc.2",
     "compression": "^1.7.4",
+    "cors": "^2.8.5",
     "dotenv": "^7.0.0",
     "ejs": "^2.6.1",
     "eslint-plugin-prettier": "^3.1.2",
@@ -55,7 +53,8 @@
     "prettier": "^1.19.1",
     "react-step-wizard": "^5.1.0",
     "sequelize": "^5.18.4",
-    "sequelize-cli": "^5.5.0"
+    "sequelize-cli": "^5.5.0",
+    "validate.js": "^0.13.1"
   },
   "devDependencies": {
     "nodemon": "^1.19.0"

test/olxScrapeTest.js

@@ -9,7 +9,7 @@ if (urlToScrape) {
 
   (async () => {
     const data = await crawler.scrapeAd(urlToScrape);
-    console.log("Scraped data:", data);
+    console.log(data);
   })();
 } else {
   console.log("No URL to scrape. Use like this : ");

test/saljicScrapeTest.js

@@ -1,17 +0,0 @@
-"use strict";
-
-const saljicCrawler = require("../app/crawler/specificCrawlers/saljic");
-
-const urlToScrape = process.argv[2] || undefined;
-
-if (urlToScrape) {
-  const crawler = new saljicCrawler();
-
-  (async () => {
-    const data = await crawler.scrapeAd(urlToScrape);
-    console.log("Scraped data:", data);
-  })();
-} else {
-  console.log("No URL to scrape. Use like this : ");
-  console.log("npm run test-saljic-scraper -- URL_TO_SCRAPE");
-}