Olx added preflight check of available concurrent req.

Saljic crawler changed substring call.

See merge request saburly/marketalarm/web!95

app/common/enums.js

@@ -304,7 +304,6 @@ const AD_AGENCY = {
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
   AKTIDO: "AKTIDO",
-  KIVI: "KIVI",
   SALJIC: "SALJIC"
 };
 

app/common/publishEnums.js

@@ -1,492 +0,0 @@
-const {
-  AD_CATEGORY,
-  ACCESS_ROAD_TYPE,
-  HEATING_TYPE,
-  FURNISHING_TYPE
-} = require("./enums");
-
-const BASIC_BOOLEAN_PUBLISH = [
-  {
-    dbField: "newBuilding",
-    title: "Novogradnja",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "balcony",
-    title: "Balkon",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-
-  {
-    dbField: "elevator",
-    title: "Lift",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "recentlyAdapted",
-    title: "Nedavno adaptirano",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-const BASIC_INPUT_PUBLISH = [
-  {
-    dbField: "title",
-    title: "Naslov",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["required"]
-  },
-  {
-    dbField: "shortDescription",
-    title: "Opis",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: []
-  },
-  {
-    dbField: "price",
-    title: "Cijena (KM)",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "area",
-    title: "Površina (m\xB2)",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "gardenSize",
-    title: "Površina okućnice (m\xB2)",
-    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE],
-    constraint: ["numerical"]
-  },
-  {
-    dbField: "streetName",
-    title: "Adresa",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ],
-    constraint: []
-  },
-  {
-    dbField: "numberOfRooms",
-    title: "Broj soba",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ],
-    constraint: ["integer"]
-  },
-  {
-    dbField: "numberOfFloors",
-    title: "Broj spratova",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ],
-    constraint: ["integer"]
-  },
-  {
-    dbField: "floor",
-    title: "Sprat",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.OFFICE
-    ],
-    constraint: ["integer"]
-  }
-];
-
-const BASIC_SEGMENT_PUBLISH = [
-  {
-    dbField: "furnishingType",
-    title: "Namještaj",
-    values: Object.keys(FURNISHING_TYPE).map(key => FURNISHING_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-const ADDITIONAL_BOOLEAN_PUBLISH = [
-  {
-    dbField: "water",
-    title: "Voda",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "electricity",
-    title: "Struja",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "drainageSystem",
-    title: "Kanalizacija",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "registeredInZkBooks",
-    title: "Uknjiženo",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-
-  {
-    dbField: "parking",
-    title: "Parking",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "garage",
-    title: "Garaža",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "gas",
-    title: "Plin",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "antiTheftDoor",
-    title: "Blindirana vrata",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "airCondition",
-    title: "Klimatizirano",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "phoneConnection",
-    title: "Telefon",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "cableTV",
-    title: "Kablovska",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "internet",
-    title: "Internet",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  },
-  {
-    dbField: "basementAttic",
-    title: "Podrum-Tavan",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "storeRoom",
-    title: "Ostava",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "videoSurveillance",
-    title: "Video nadzor",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "alarm",
-    title: "Alarm",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "suitableForStudents",
-    title: "Za studente",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "includingBills",
-    title: "Uključen trošak režija",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "animalsAllowed",
-    title: "Kućni ljubimci dozvoljeni",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE
-    ]
-  },
-  {
-    dbField: "pool",
-    title: "Bazen",
-    categoriesToShow: [AD_CATEGORY.HOUSE, AD_CATEGORY.COTTAGE]
-  },
-  {
-    dbField: "exchange",
-    title: "Zamjena",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "urbanPlanPermit",
-    title: "Urbanistička dozvola",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "buildingPermit",
-    title: "Građevinska dozvola",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  }
-];
-
-const ADDITIONAL_INPUT_PUBLISH = [
-  {
-    dbField: "longDescription",
-    title: "Detaljan opis",
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  }
-];
-
-const ADDITIONAL_SEGMENT_PUBLISH = [
-  {
-    dbField: "accessRoadType",
-    title: "Pristupni put",
-    values: Object.keys(ACCESS_ROAD_TYPE).map(key => ACCESS_ROAD_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE,
-      AD_CATEGORY.LAND,
-      AD_CATEGORY.GARAGE
-    ]
-  },
-  {
-    dbField: "heatingType",
-    title: "Grijanje",
-    values: Object.keys(HEATING_TYPE).map(key => HEATING_TYPE[key]),
-    categoriesToShow: [
-      AD_CATEGORY.FLAT,
-      AD_CATEGORY.HOUSE,
-      AD_CATEGORY.APARTMENT,
-      AD_CATEGORY.COTTAGE,
-      AD_CATEGORY.OFFICE
-    ]
-  }
-];
-
-module.exports = {
-  BASIC_INPUT_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  BASIC_BOOLEAN_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH
-};

app/config/appConfig.js

@@ -9,7 +9,7 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
-const STAGING = process.env.SETTINGS !== "production";
+const STAGING = process.env.ENVIRONMENT !== "production";
 
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
@@ -34,13 +34,22 @@ const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
 
 const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
-const GOOGLE_MAP_KEY = process.env.GOOGLE_MAP_KEY || "";
+const API_MAP_KEY = process.env.API_MAP_KEY || "";
 
 const PROSTOR_LOGIN = {
   EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
   PASSWORD: process.env.PROSTOR_LOGIN_PASS
 };
 
+const USER_AGENT =
+  process.env.USER_AGENT ||
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
+
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+const NUMBER_OF_CONCURRENT_REQ_SCRAPER_API =
+  parseInt(process.env.NUMBER_OF_CONCURRENT_REQ_SCRAPER_API) || 10;
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -51,8 +60,12 @@ module.exports = {
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
   PRINT_CRAWLER_DEBUG,
-  GOOGLE_MAP_KEY,
+  API_MAP_KEY,
   STAGING,
   CHECK_UP_DAYS,
-  PROSTOR_LOGIN
+  PROSTOR_LOGIN,
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
 };

app/controllers/publishRealEstate.js

@@ -1,327 +0,0 @@
-const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
-const { bulkUpsertKiviPhotos } = require("../helpers/db/kiviOriginalAdsPhotos");
-const { currentKiviRealEstate } = require("../helpers/url");
-
-const validate = require("validate.js");
-
-const {
-  AD_CATEGORY,
-  FURNISHING_TYPE,
-  ACCESS_ROAD_TYPE,
-  HEATING_TYPE
-} = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
-
-const {
-  BASIC_BOOLEAN_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH,
-  BASIC_INPUT_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH
-} = require("../common/publishEnums");
-
-const getPublishInputs = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const pageTitle = "Podaci o nekretnini";
-
-  const {
-    price,
-    area,
-    adType,
-    realEstateType,
-    locationLat,
-    locationLong,
-    accessRoadType,
-    heatingType,
-    balcony,
-    newBuilding,
-    elevator,
-    recentlyAdapted,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit,
-    furnishingType,
-    shortDescription,
-    streetName,
-    title,
-    longDescription
-  } = realEstate;
-  const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
-
-  // TODO: Maybe this is slow, pay attention to this
-  const filterInputs = filterObject => {
-    const filterCategories = filterObject.categoriesToShow;
-    return filterCategories.indexOf(category) !== -1;
-  };
-  //Boolean inputs to be shown on Basic Data tab
-  const basicBooleanPublishInputs = BASIC_BOOLEAN_PUBLISH.filter(filterInputs);
-  const basicBooleanPublishValues = {
-    balcony,
-    elevator,
-    newBuilding,
-    recentlyAdapted
-  };
-  //Boolean inputs to be shown on Additional Data tab
-  const additionalBooleanPublishInputs = ADDITIONAL_BOOLEAN_PUBLISH.filter(
-    filterInputs
-  );
-  const additionalBooleanPublishValues = {
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit
-  };
-  //Segment select inputs to be shown on Basic Data tab
-  const basicSegmentSelectInputs = BASIC_SEGMENT_PUBLISH.filter(filterInputs);
-  const basicSegmentSelectValues = {
-    furnishingType
-  };
-  //Segment select inputs to be shown on Additional Data tab
-  const additionalSegmentSelectInputs = ADDITIONAL_SEGMENT_PUBLISH.filter(
-    filterInputs
-  );
-  const additionalSegmentSelectValues = {
-    accessRoadType,
-    heatingType
-  };
-  //Input text type inputs to be shown on Basic Data tab
-  const basicInputInputs = BASIC_INPUT_PUBLISH.filter(filterInputs);
-  const basicInputValues = {
-    price,
-    area,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    title,
-    shortDescription,
-    streetName
-  };
-  //Input type textare to be shown on Additional Data
-  const additionalInputInputs = ADDITIONAL_INPUT_PUBLISH.filter(filterInputs);
-  const additionalInputValues = {
-    longDescription
-  };
-
-  res.render("publishRealEstate", {
-    title: pageTitle,
-    basicBooleanPublishInputs,
-    basicBooleanPublishValues,
-    additionalBooleanPublishInputs,
-    additionalBooleanPublishValues,
-    basicSegmentSelectInputs,
-    basicSegmentSelectValues,
-    additionalSegmentSelectInputs,
-    additionalSegmentSelectValues,
-    basicInputInputs,
-    basicInputValues,
-    additionalInputInputs,
-    additionalInputValues,
-    validate: validate
-  });
-};
-
-const postPublishInputs = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  if (!kiviOriginal || !kiviOriginal.kiviAdId) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const nextStepPage = req.query.nextStep || "/uspjesnaobjava";
-
-  //Request body
-  //console.log("Body:", req.body);
-
-  const balcony = req.body.balcony === "on";
-  const elevator = req.body.elevator === "on";
-  const newBuilding = req.body.newBuilding === "on";
-  const recentlyAdapted = req.body.recentlyAdapted === "on";
-  const water = req.body.water === "on";
-  const electricity = req.body.electricity === "on";
-  const drainageSystem = req.body.drainageSystem === "on";
-  const registeredInZkBooks = req.body.registeredInZkBooks === "on";
-  const parking = req.body.parking === "on";
-  const garage = req.body.garage === "on";
-  const gas = req.body.gas === "on";
-  const antiTheftDoor = req.body.antiTheftDoor === "on";
-  const airCondition = req.body.airCondition === "on";
-  const phoneConnection = req.body.phoneConnection === "on";
-  const cableTV = req.body.cableTV === "on";
-  const internet = req.body.internet === "on";
-  const basementAttic = req.body.basementAttic === "on";
-  const storeRoom = req.body.storeRoom === "on";
-  const videoSurveillance = req.body.videoSurveillance === "on";
-  const alarm = req.body.alarm === "on";
-  const suitableForStudents = req.body.suitableForStudents === "on";
-  const includingBills = req.body.includingBills === "on";
-  const animalsAllowed = req.body.animalsAllowed === "on";
-  const pool = req.body.pool === "on";
-  const exchange = req.body.exchange === "on";
-  const urbanPlanPermit = req.body.urbanPlanPermit === "on";
-  const buildingPermit = req.body.buildingPermit === "on";
-
-  const furnishingType = req.body.furnishingType;
-  //VALIDACIJA TAKO POTVRDITI DA JE ISPRAVNA VRIJEDNOST
-  /* if (!FURNISHING_TYPE[furnishingType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  } */
-  const accessRoadType = req.body.accessRoadType;
-  /*if (!ACCESS_ROAD_TYPE[accessRoadType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  } */
-  const heatingType = req.body.heatingType;
-  /*if (!HEATING_TYPE[heatingType]) {
-    res.render("notFound", { title: " Greška !" });
-    return;
-  }*/
-
-  const price = parseFloat(req.body.price) || null;
-  const area = parseFloat(req.body.area) || null;
-  const gardenSize = parseFloat(req.body.gardenSize) || null;
-  const numberOfRooms = parseInt(req.body.numberOfRooms) || null;
-  const numberOfFloors = parseInt(req.body.numberOfFloors) || null;
-  const floor = parseInt(req.body.floor) || null;
-  const title = req.body.title || "";
-  const shortDescription = req.body.shortDescription || "";
-  const streetName = req.body.streetName || "";
-  const longDescription = req.body.longDescription || "";
-
-  const locationLat = req.body.lat || null;
-  const locationLong = req.body.lng || null;
-  //Contact email saved in other table
-  const contactEmail = req.body.email || "";
-  //Image urls are stored in new table
-  const imageUrls =
-    req.body.imageUrls.split("|").filter(url => url !== "") || [];
-  const imageUrlsData = imageUrls.map(url => {
-    return {
-      kiviAdId: kiviOriginal.kiviAdId,
-      photoUrl: url
-    };
-  });
-  const savedImageUrls = await bulkUpsertKiviPhotos(imageUrlsData);
-
-  realEstate.balcony = balcony;
-  realEstate.elevator = elevator;
-  realEstate.newBuilding = newBuilding;
-  realEstate.recentlyAdapted = recentlyAdapted;
-  realEstate.water = water;
-  realEstate.electricity = electricity;
-  realEstate.drainageSystem = drainageSystem;
-  realEstate.registeredInZkBooks = registeredInZkBooks;
-  realEstate.parking = parking;
-  realEstate.garage = garage;
-  realEstate.gas = gas;
-  realEstate.antiTheftDoor = antiTheftDoor;
-  realEstate.airCondition = airCondition;
-  realEstate.phoneConnection = phoneConnection;
-  realEstate.cableTV = cableTV;
-  realEstate.internet = internet;
-  realEstate.basementAttic = basementAttic;
-  realEstate.storeRoom = storeRoom;
-  realEstate.videoSurveillance = videoSurveillance;
-  realEstate.alarm = alarm;
-  realEstate.suitableForStudents = suitableForStudents;
-  realEstate.includingBills = includingBills;
-  realEstate.animalsAllowed = animalsAllowed;
-  realEstate.pool = pool;
-  realEstate.exchange = exchange;
-  realEstate.urbanPlanPermit = urbanPlanPermit;
-  realEstate.buildingPermit = buildingPermit;
-
-  realEstate.furnishingType = furnishingType;
-  realEstate.accessRoadType = accessRoadType;
-  realEstate.heatingType = heatingType;
-
-  realEstate.price = price;
-  realEstate.area = area;
-  realEstate.gardenSize = gardenSize;
-  realEstate.numberOfRooms = numberOfRooms;
-  realEstate.numberOfFloors = numberOfFloors;
-  realEstate.floor = floor;
-  realEstate.title = title;
-  realEstate.shortDescription = shortDescription;
-  realEstate.streetName = streetName;
-
-  realEstate.longDescription = longDescription;
-
-  realEstate.locationLat = locationLat;
-  realEstate.locationLong = locationLong;
-
-  kiviOriginal.email = contactEmail;
-
-  //console.log("realEstate", realEstate);
-
-  await realEstate.save();
-
-  await kiviOriginal.save();
-
-  res.redirect(nextStepPage);
-};
-
-module.exports = {
-  getPublishInputs,
-  postPublishInputs
-};

app/controllers/publishRealEstateTypes.js

@@ -1,105 +0,0 @@
-const { currentKiviRealEstate } = require("../helpers/url");
-const {
-  createRealEstate,
-  findRealEstateByAgencyId
-} = require("../helpers/db/realEstate");
-const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
-
-const { AD_CATEGORY, AD_TYPE, AD_AGENCY } = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
-
-const getPublishTypes = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  const title = "Koju nekretninu nudite?";
-  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
-  const labelAdType = ["Prodaj", "Iznajmi"];
-
-  if (
-    realEstate &&
-    realEstate.adType &&
-    realEstate.adType === AD_TYPE.AD_TYPE_RENT.stringId
-  ) {
-    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
-  }
-  const realEstateTypes = Object.keys(AD_CATEGORY)
-    .map(category => AD_CATEGORY[category])
-    .filter(category => category.title);
-
-  res.render("realEstateType", {
-    selectedAdType,
-    labelAdType,
-    realEstateTypes,
-    title,
-    AD_TYPE
-  });
-};
-
-const postPublishTypes = async (req, res) => {
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId);
-
-  const adType = parseInt(req.body.adType);
-
-  const adTypeStringIds = {
-    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
-    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
-  };
-
-  const adTypeStringId =
-    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
-
-  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
-    category => !!AD_CATEGORY[category].title
-  );
-
-  const selectedRealEstateType = req.body.realEstateType || null;
-  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const nextStepPage = req.query.nextStep || "podacionekretnini";
-
-  let nextStepUrl = "";
-  if (kiviOriginal && kiviOriginal.kiviAdId && realEstate && realEstate.id) {
-    //
-    nextStepUrl = `/${nextStepPage}/${kiviOriginal.kiviAdId}`;
-
-    realEstate.adType = adTypeStringId;
-    realEstate.realEstateType = selectedRealEstateType;
-    //Url override
-    realEstate.url = `${APP_URL}/preglednekretnine/${realEstate.id}`;
-
-    await realEstate.save();
-  } else {
-    try {
-      const newKiviOriginal = await createKiviOriginal({
-        email: ""
-      });
-      const newKiviAdViewUrl = `${APP_URL}/preglednekretnine/${realEstate.id}`;
-
-      const newRealEstate = await createRealEstate({
-        adType: adTypeStringId,
-        realEstateType: selectedRealEstateType,
-        url: newKiviAdViewUrl,
-        originAgencyName: AD_AGENCY.KIVI,
-        agencyObjectId: newKiviOriginal.kiviAdId
-      });
-
-      nextStepUrl = `/${nextStepPage}/${newKiviOriginal.kiviAdId}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  }
-  res.redirect(nextStepUrl);
-};
-
-module.exports = {
-  getPublishTypes,
-  postPublishTypes
-};

app/controllers/publishSuccess.js

@@ -1,8 +0,0 @@
-const publishSuccess = async (req, res) => {
-  const title = "Uspjeh!";
-  res.render("publishSuccess", { title });
-};
-
-module.exports = {
-  publishSuccess
-};

app/controllers/realEstateFilters.js

@@ -122,8 +122,6 @@ const getFilters = async (req, res) => {
 };
 
 const postFilters = async (req, res) => {
-  //
-  console.log("postFilters");
   const searchRequest = await currentSearchRequest(req);
 
   if (!searchRequest || !searchRequest.dataValues) {

app/controllers/realEstateTypes.js

@@ -8,7 +8,6 @@ const getRealEstateTypes = async (req, res) => {
 
   const title = "Koju nekretninu tražite?";
   let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
-  const labelAdType = [AD_TYPE.AD_TYPE_SALE.title, AD_TYPE.AD_TYPE_RENT.title];
   if (
     searchRequest &&
     searchRequest.adType &&
@@ -22,7 +21,6 @@ const getRealEstateTypes = async (req, res) => {
 
   res.render("realEstateType", {
     selectedAdType,
-    labelAdType,
     realEstateTypes,
     title,
     AD_TYPE

app/controllers/viewRealEstate.js

@@ -1,202 +0,0 @@
-const { findRealEstateByAgencyId } = require("../helpers/db/realEstate");
-const { findPhotosForKiviAd } = require("../helpers/db/kiviOriginalAdsPhotos");
-const { currentKiviRealEstate, currentRealEstate } = require("../helpers/url");
-
-const {
-  BASIC_BOOLEAN_PUBLISH,
-  BASIC_SEGMENT_PUBLISH,
-  ADDITIONAL_BOOLEAN_PUBLISH,
-  ADDITIONAL_SEGMENT_PUBLISH,
-  BASIC_INPUT_PUBLISH,
-  ADDITIONAL_INPUT_PUBLISH
-} = require("../common/publishEnums");
-
-const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
-
-const getViewRealEstate = async (req, res) => {
-  //Variation if we acces to real estate previews via kiviAdId
-  /*
-  const kiviOriginal = await currentKiviRealEstate(req);
-
-  if (!kiviOriginal || !kiviOriginal.kiviAdId) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-  const realEstate = await findRealEstateByAgencyId(kiviOriginal.kiviAdId); */
-
-  const realEstate = await currentRealEstate(req);
-
-  if (!realEstate || !realEstate.dataValues) {
-    res.render("notFound", { title: " " });
-    return;
-  }
-
-  const pageTitle = "Pregled nekretnine";
-
-  const {
-    price,
-    area,
-    adType,
-    agencyObjectId,
-    realEstateType,
-    locationLat,
-    locationLong,
-    accessRoadType,
-    heatingType,
-    balcony,
-    newBuilding,
-    elevator,
-    recentlyAdapted,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit,
-    furnishingType,
-    shortDescription,
-    streetName,
-    title,
-    longDescription
-  } = realEstate;
-  //Categorize all database values by value type - input, boolean or segment selected
-  const allInputValues = {
-    price,
-    area,
-    gardenSize,
-    numberOfRooms,
-    numberOfFloors,
-    floor,
-    title,
-    shortDescription,
-    streetName,
-    longDescription
-  };
-  const allBooleanValues = {
-    balcony,
-    elevator,
-    newBuilding,
-    recentlyAdapted,
-    water,
-    electricity,
-    drainageSystem,
-    registeredInZkBooks,
-    parking,
-    garage,
-    gas,
-    antiTheftDoor,
-    airCondition,
-    phoneConnection,
-    cableTV,
-    internet,
-    basementAttic,
-    storeRoom,
-    videoSurveillance,
-    alarm,
-    suitableForStudents,
-    includingBills,
-    animalsAllowed,
-    pool,
-    exchange,
-    urbanPlanPermit,
-    buildingPermit
-  };
-
-  const allSegmentSelectedValues = {
-    furnishingType,
-    accessRoadType,
-    heatingType
-  };
-
-  //We need titles of fields ex Balkon, Novogradnja
-  const ALL_BOOLEAN_FIELDS = [
-    ...BASIC_BOOLEAN_PUBLISH,
-    ...ADDITIONAL_BOOLEAN_PUBLISH
-  ];
-  const ALL_INPUT_FIELDS = [
-    ...BASIC_INPUT_PUBLISH,
-    ...ADDITIONAL_INPUT_PUBLISH
-  ];
-  const ALL_SEGMENT_FIELDS = [
-    ...BASIC_SEGMENT_PUBLISH,
-    ...ADDITIONAL_SEGMENT_PUBLISH
-  ];
-
-  //On view add page we will show only values that are not - null, or "", or undefined
-  const forShowing = value => {
-    return value !== false && value !== null && value !== "";
-  };
-  //Filter all values to be shown on page or not
-  //For showing on page we also need title ex. "Balkon"
-  const booleanFields = ALL_BOOLEAN_FIELDS.filter(object => {
-    return forShowing(allBooleanValues[object.dbField]);
-  });
-  const inputFields = ALL_INPUT_FIELDS.filter(object => {
-    return forShowing(allInputValues[object.dbField]);
-  });
-  const segmentFields = ALL_SEGMENT_FIELDS.filter(object => {
-    return forShowing(allSegmentSelectedValues[object.dbField]);
-  });
-
-  //Photo urls from Google storage bucket
-  const kiviAdId = agencyObjectId;
-  const urlGooglePrefix =
-    "https://storage.cloud.google.com/marketalarm-photos/";
-  const realEstatePhotosData = await findPhotosForKiviAd(kiviAdId);
-  const realEstatePhotosUrls = realEstatePhotosData.map(row => {
-    return urlGooglePrefix + row.dataValues.photoUrl;
-  });
-
-  const showRealEstateType = AD_CATEGORY[realEstateType].title.toUpperCase();
-  let showAdType = "";
-  switch (adType) {
-    case AD_TYPE.AD_TYPE_SALE.stringId:
-      showAdType = AD_TYPE.AD_TYPE_SALE.title.toUpperCase();
-      break;
-    case AD_TYPE.AD_TYPE_RENT.stringId:
-      showAdType = AD_TYPE.AD_TYPE_RENT.title.toUpperCase();
-      break;
-    default:
-      showAdType = "-";
-      break;
-  }
-
-  res.render("viewRealEstate", {
-    title: pageTitle,
-    booleanFields,
-    inputFields,
-    allInputValues,
-    segmentFields,
-    allSegmentSelectedValues,
-    locationLat,
-    locationLong,
-    showAdType,
-    showRealEstateType,
-    realEstatePhotosUrls
-  });
-};
-
-module.exports = {
-  getViewRealEstate
-};

app/controllers/welcome.js

@@ -1,8 +1,6 @@
 const { createSearchRequest } = require("../helpers/db/searchRequest");
-const { createRealEstate } = require("../helpers/db/realEstate");
-const { createKiviOriginal } = require("../helpers/db/kiviOriginal");
-const { AD_TYPE, AD_CATEGORY, AD_AGENCY } = require("../common/enums");
-const { APP_URL } = require("../config/appConfig");
+
+const { AD_TYPE, AD_CATEGORY } = require("../common/enums");
 
 const getWelcome = (req, res) => {
   res.render("welcome", {
@@ -13,55 +11,7 @@ const getWelcome = (req, res) => {
 
 const postWelcome = async (req, res) => {
   const adType = parseInt(req.body.adType);
-  const publishAdType = parseInt(req.body.publishAdType);
 
-  let nextStepUrl = "";
-
-  if (adType) {
-    const adTypeStringId = getAdTypeString(adType);
-    try {
-      const newSearchRequest = await createSearchRequest({
-        adType: adTypeStringId,
-        realEstateType: AD_CATEGORY.FLAT.id
-      });
-
-      nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  } else if (publishAdType) {
-    const adTypeStringId = getAdTypeString(publishAdType);
-
-    try {
-      //First we create new Kivi Ad Original object in db then new Real Estate
-      //Problem with id-s
-      const newKiviOriginal = await createKiviOriginal({
-        email: ""
-      });
-      //Temporary url because we have cyclic id call - need to override for safety measures
-      const newKiviAdViewUrl = `${APP_URL}/preglednekretnine/${newKiviOriginal.kiviAdId}`;
-
-      const newRealEstate = await createRealEstate({
-        adType: adTypeStringId,
-        realEstateType: AD_CATEGORY.FLAT.id,
-        url: newKiviAdViewUrl,
-        originAgencyName: AD_AGENCY.KIVI,
-        agencyObjectId: newKiviOriginal.kiviAdId
-      });
-
-      nextStepUrl = `/objavinekretninu/${newKiviOriginal.kiviAdId}`;
-    } catch (error) {
-      console.log(error);
-      nextStepUrl = `/`;
-    }
-  }
-
-  res.redirect(nextStepUrl);
-};
-
-//--- Helper function
-const getAdTypeString = adType => {
   const adTypeStringIds = {
     [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
     [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
@@ -70,7 +20,20 @@ const getAdTypeString = adType => {
   const adTypeStringId =
     adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
 
-  return adTypeStringId;
+  let nextStepUrl = "";
+  try {
+    const newSearchRequest = await createSearchRequest({
+      adType: adTypeStringId,
+      realEstateType: AD_CATEGORY.FLAT.id
+    });
+
+    nextStepUrl = `/vrstanekretnine/${newSearchRequest.id}`;
+  } catch (error) {
+    console.log(error);
+    nextStepUrl = `/`;
+  }
+
+  res.redirect(nextStepUrl);
 };
 
 module.exports = {

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/olx.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -18,7 +18,9 @@ const {
 
 const {
   DEFAULT_TIMEZONE,
-  PRINT_CRAWLER_DEBUG
+  PRINT_CRAWLER_DEBUG,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API,
+  SCRAPER_API_KEY
 } = require("../../config/appConfig");
 
 const OLX_ENUMS = {
@@ -44,6 +46,8 @@ const OLX_ENUMS = {
 
 const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
 
+const scraperapiClient = require("scraperapi-sdk")(SCRAPER_API_KEY);
+
 class OlxCrawler {
   constructor(
     savers = [],
@@ -190,12 +194,40 @@ class OlxCrawler {
       let actualNoOfResults =
         hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
 
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      const scrapedData = [];
+      for (
+        let i = 0;
+        i <= actualNoOfResults;
+        i = i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+      ) {
+        const concurrentUrlsToScrape = hrefs.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+        //Before it send n req to scraperAPI it send preflight request to check if we have enough concurrent req availabe
+        //It does not send "real" req until approven internaly
+        let availableConcurrentReqSlots = false;
+        do {
+          availableConcurrentReqSlots = await this.checkAvailableConcurrentReqSlots(
+            concurrentUrlsToScrape.length
+          );
+        } while (availableConcurrentReqSlots !== true);
+        //
+        console.log(
+          `OLX - Sending requests from ${i} to ${i +
+            NUMBER_OF_CONCURRENT_REQ_SCRAPER_API}.`
+        );
+        console.log(`OLX - Urls sent to scrape: `, concurrentUrlsToScrape);
+        //
+        const concurrentReqScraperApi = concurrentUrlsToScrape.map(url =>
+          this.scrapeAd(url)
+        );
+
+        const concurrentReqData = await Promise.all(concurrentReqScraperApi);
+
+        concurrentReqData.forEach(reqData => scrapedData.push(reqData));
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
       const filteredScrapedData = scrapedData.filter(adData => !!adData);
       return filteredScrapedData;
     } catch (e) {
@@ -206,6 +238,7 @@ class OlxCrawler {
 
   async scrapeAd(url) {
     // console.log("Scraping : ", url);
+
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
@@ -238,15 +271,28 @@ class OlxCrawler {
 
       //====== PRICE DETECTION AND EXTRACTION =====
       let price = null;
-      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      let normalPrice = null;
+      let urgentPrice = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)")
+        .text()
+        .trim();
       const urgentPriceValue = $(
         "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
       )
         .text()
         .trim();
+      //For cases where price is given in discount manner - different from default parsing
+      const discountPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div.op.pop > p"
+      )
+        .text()
+        .trim();
 
       if (normalPriceValue && normalPriceValue.length > 0) {
-        price = normalPriceValue;
+        normalPrice = normalPriceValue
+          .replace(/\r\n|\n|\r/gm, "")
+          .replace("KM", "")
+          .trim();
         if (
           $("#pc > p.n")
             .text()
@@ -256,21 +302,35 @@ class OlxCrawler {
         } else {
           status = AD_STATUS.STATUS_NORMAL;
         }
-      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
-        const priceValues = urgentPriceValue.split("KM");
+      } else if (discountPriceValue && discountPriceValue.length > 0) {
+        status = AD_STATUS.STATUS_URGENT;
+        const priceValues = discountPriceValue.split("KM");
+        normalPrice = priceValues[0].trim();
+      } else {
+        console.log("Body:", body);
+        throw { message: "Can't find normal price" };
+      }
+      if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.replace("Cijena", "").split("KM");
         //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
-        if (priceValues.length > 1) {
-          price = priceValues[1].trim();
-          status = AD_STATUS.STATUS_DISCOUNTED;
+        if (priceValues.length > 0) {
+          if (priceValues[0].trim().indexOf("Hitno") != -1) {
+            urgentPrice = priceValues[0].replace("Hitno", "").trim();
+            status = AD_STATUS.STATUS_URGENT;
+          } else {
+            urgentPrice = priceValues[0].trim();
+          }
+        } else if (discountPriceValue && discountPriceValue.length > 0) {
+          status = AD_STATUS.STATUS_URGENT;
+          const priceValues = discountPriceValue.split("KM");
+          urgentPrice = priceValues[1].trim();
         } else {
           throw { message: "Can't find urgent price" };
         }
-      } else {
-        throw {
-          message: "Can't find price (it is not normal nor urgent price ?)"
-        };
       }
 
+      price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
+
       //====== OTHER AD INFORMATION ===============
       let adType = null;
       let olxId = null;
@@ -278,7 +338,7 @@ class OlxCrawler {
 
       let otherInformationDivId;
       //We need to locate DIV ID where other information are stored
-      for (let possibleId = 10; possibleId <= 20; possibleId++) {
+      for (let possibleId = 1; possibleId <= 30; possibleId++) {
         const adTypeFieldTitle = $(
           `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
         )
@@ -655,6 +715,7 @@ class OlxCrawler {
     } catch (e) {
       console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
+
     return null;
   }
 
@@ -768,6 +829,9 @@ class OlxCrawler {
     if (!priceText) {
       return NaN;
     }
+    if (priceText === "Po dogovoru") {
+      return null;
+    }
     const formattedPriceText = priceText.replace(".", "").replace(",", ".");
     return parseFloat(formattedPriceText);
   }
@@ -867,8 +931,28 @@ class OlxCrawler {
     console.log("sprat = NEPOZNATO [", floorText, "]");
     return null;
   }
+  async checkAvailableConcurrentReqSlots(numberOfNeededConcurrentReq) {
+    try {
+      const scraperApiAccountInfo = await scraperapiClient.account();
+      const numberOfUsedConcurrentReq =
+        scraperApiAccountInfo.concurrentRequests;
+      const limitOfConcurrentReq = scraperApiAccountInfo.concurrencyLimit;
+      //Buffer of requests to prevent errors with prefligh requests
+      const bufferNumberOfReq = 3;
+      const numberOfAvailableConcurrentReq =
+        limitOfConcurrentReq - bufferNumberOfReq - numberOfUsedConcurrentReq;
+      if (numberOfNeededConcurrentReq <= numberOfAvailableConcurrentReq) {
+        return true;
+      } else {
+        return false;
+      }
+    } catch (err) {
+      return false;
+    }
+  }
 
   async sleep(ms) {
+    // console.log("Sleep for:", ms);
     return new Promise(resolve => setTimeout(resolve, ms));
   }
 

app/crawler/specificCrawlers/prostor.js

@@ -1,9 +1,10 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
+const nodeFetch = require("node-fetch");
 
 const {
   AD_TYPE,
@@ -191,19 +192,13 @@ class ProstorCrawler {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
 
     //Status information is given already in realestate list
-    //For VIP Ads status ='' canot be used, but no VIP ads are crawled
-    //We will make "fake" vip ad for RE that have size=55
-    //It is weird because yesterday it said 'VIP ponuda' ???
-    const adStatus =
-      size === "55"
-        ? ProstorCrawler.getStatusId("VIP ponuda")
-        : ProstorCrawler.getStatusId(status);
+    const adStatus = ProstorCrawler.getStatusId(status);
 
     const url = `https://prostor.ba${link}`;
 
     // console.log("[PROSTOR] Scraping : ", url);
     try {
-      const adPageSource = await fetch(url, {
+      const adPageSource = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await adPageSource.text();
@@ -433,7 +428,7 @@ class ProstorCrawler {
     }
 
     try {
-      const res = await fetch(url, {
+      const res = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await res.text();
@@ -597,7 +592,7 @@ class ProstorCrawler {
     formData.append("email", PROSTOR_LOGIN.EMAIL);
     formData.append("password", PROSTOR_LOGIN.PASSWORD);
 
-    return fetch("https://prostor.ba/moj-prostor/prijava", {
+    return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
       method: "POST",
       body: formData,
       headers: { Cookie: prostorCookie }
@@ -624,9 +619,12 @@ class ProstorCrawler {
       });
   }
   async getCookies() {
-    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
-      headers: { Cookie: "" }
-    });
+    const getResponse = await nodeFetch(
+      "https://prostor.ba/moj-prostor/prijava",
+      {
+        headers: { Cookie: "" }
+      }
+    );
     const raw = getResponse.headers.raw()["set-cookie"];
     const cookie = raw
       .map(datastring => {

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -399,7 +399,9 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
           };
         }
 
@@ -410,7 +412,9 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 
@@ -16,7 +16,8 @@ const {
 
 const {
   PRINT_CRAWLER_DEBUG,
-  DEFAULT_TIMEZONE
+  DEFAULT_TIMEZONE,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
 } = require("../../config/appConfig");
 const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
 
@@ -84,6 +85,7 @@ class SaljicCrawler {
         for (const [index, { value: singlePageResult }] of entries) {
           if (singlePageResult) {
             const saveResults = await this.saveCrawledResults(singlePageResult);
+
             const { newRecords } = saveResults;
 
             newRealEstates.push(...newRecords);
@@ -203,13 +205,32 @@ class SaljicCrawler {
           ? hrefsAbs.length
           : maxResultsPerPage;
 
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+      const scrapedData = [];
+      for (
+        let i = 0;
+        i <= actualNoOfResults;
+        i = i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+      ) {
+        const concurrentUrlsToScrape = hrefsAbs.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+
+        const concurrentAdTypesOfReq = adTypes.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+
+        const concurrentReqScraperApi = concurrentUrlsToScrape.map(
+          (url, index) => this.scrapeAd(url, concurrentAdTypesOfReq[index])
+        );
+        const concurrentReqData = await Promise.all(concurrentReqScraperApi);
+
+        concurrentReqData.forEach(reqData => scrapedData.push(reqData));
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
       const filteredScrapedData = scrapedData.filter(adData => !!adData);
+
       return filteredScrapedData;
     } catch (e) {
       console.error("[SALJIC] Exception caught:" + e);
@@ -217,22 +238,28 @@ class SaljicCrawler {
     }
   }
 
-  async scrapeAd(url, adType) {
-    // console.log("[SALJIC] Scraping : ", url);
+  async scrapeAd(url, adTypeAttribute) {
+    //console.log("[SALJIC] Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      //Throws error if req to Scraper API proxy wasn't succesful and responds with error
+      if (body.indexOf("<html>") === -1) {
+        throw { message: "Scraper API server error." };
+      }
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
+      const agencyObjectId = url
+        ? parseInt(url.substring(46, url.length))
+        : null;
 
       //Extracting main properties
       const propertySelectors = {
         title:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
+          "div.content-wrap > div.container.clearfix.wpc > div.col-md-8.nobottommargin > div.single-post.nobottommargin > div.entry.clearfix > div.entry-title > h2",
         price:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
         streetName:
@@ -243,6 +270,7 @@ class SaljicCrawler {
         latAndLong:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
       };
+
       const title = $(propertySelectors.title)
         .text()
         .replace(/(\r\n|\n|\r)/gm, "")
@@ -272,14 +300,15 @@ class SaljicCrawler {
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
+      let tmpLatLong;
+      let latText;
+      let longText;
+
+      if (latAndLongSrc && latAndLongSrc.indexOf("openstreetmap") !== -1) {
+        tmpLatLong = latAndLongSrc.split("marker=")[1];
+        latText = tmpLatLong.split("%2C")[0];
+        longText = tmpLatLong.split("%2C")[1];
+      }
       const locationLat = parseFloat(latText) || null;
       const locationLong = parseFloat(longText) || null;
 
@@ -328,11 +357,11 @@ class SaljicCrawler {
       let numberOfViewsKivi = null;
       let streetNumber = 0;
       let adStatus = status;
-      let shortDescription = descriptions.substring(
-        0,
-        descriptions.indexOf(".")
-      );
-      let longDescription = descriptions;
+      let adType = adTypeAttribute;
+      let shortDescription = descriptions
+        ? descriptions.substring(0, descriptions.indexOf("."))
+        : "";
+      let longDescription = descriptions || "";
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -343,10 +372,14 @@ class SaljicCrawler {
           .replace(/[\n\r\t]/gm, "")
           .trim();
 
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldTitle = mainField
+          ? mainField.substring(0, mainField.indexOf("  "))
+          : "";
         const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
+          ? mainField
+              .substring(mainField.indexOf("  "), mainField.length)
+              .trim()
+          : "";
 
         switch (mainFieldTitle) {
           case "Površina":
@@ -408,6 +441,7 @@ class SaljicCrawler {
               additionalField.length
             )
             .trim();
+
           realEstateType = this.getAdCategoryId(categoryTmp);
         } else {
           switch (additionalField) {
@@ -498,6 +532,11 @@ class SaljicCrawler {
       const region = "";
       const entity = "";
       const country = "";
+      //Throws error if realEstateType is null - not read. Still dont know why?
+      if (realEstateType === null) {
+        console.log("Body:", body);
+        throw { message: "Couldn't read real estate type." };
+      }
 
       const data = {
         url,
@@ -567,6 +606,7 @@ class SaljicCrawler {
     } catch (e) {
       console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
+
     return null;
   }
 

app/helpers/db/kiviOriginal.js

@@ -1,20 +0,0 @@
-"use strict";
-const db = require("../../models/index");
-const sequelize = require("sequelize");
-
-const createKiviOriginal = async (kiviAdFields = {}) => {
-  return await db.KiviOriginal.create(kiviAdFields);
-};
-
-const getKiviOriginalById = async id => {
-  try {
-    return db.KiviOriginal.findByPk(id);
-  } catch (error) {
-    console.log("kiviOriginal.js", error);
-    return null;
-  }
-};
-module.exports = {
-  createKiviOriginal,
-  getKiviOriginalById
-};

app/helpers/db/kiviOriginalAdsPhotos.js

@@ -1,31 +0,0 @@
-"use strict";
-const db = require("../../models/index");
-const sequelize = require("sequelize");
-
-const bulkUpsertKiviPhotos = async kiviPhotosData => {
-  try {
-    return await db.KiviOriginalAdsPhotos.bulkCreate(kiviPhotosData, {
-      ignoreDuplicates: true
-    });
-  } catch (e) {
-    console.log("Error bulk upserting kiviOriginalAdsPhotos : ", e);
-  }
-};
-
-const findPhotosForKiviAd = async id => {
-  try {
-    return db.KiviOriginalAdsPhotos.findAll({
-      where: {
-        kiviAdId: id
-      }
-    });
-  } catch (error) {
-    console.log("kiviOriginalAdsPhotos.js", error);
-    return null;
-  }
-};
-
-module.exports = {
-  bulkUpsertKiviPhotos,
-  findPhotosForKiviAd
-};

app/helpers/db/realEstate.js

@@ -77,28 +77,9 @@ const bulkUpsertRealEstates = async realEstateData => {
 };
 
 const getRealEstateById = async id => {
-  try {
-    return db.RealEstate.findByPk(id);
-  } catch (error) {
-    console.log("realEstate.js", error);
-    return null;
-  }
+  return db.RealEstate.findByPk(id);
 };
 
-const createRealEstate = async (realEstateFields = {}) => {
-  return await db.RealEstate.create(realEstateFields);
-};
-
-const findRealEstateByAgencyId = async kiviId => {
-  try {
-    return db.RealEstate.findOne({
-      where: { agencyObjectId: kiviId }
-    });
-  } catch (error) {
-    console.log("realEstate.js", error);
-    return null;
-  }
-};
 const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const {
     priceMin,
@@ -351,10 +332,14 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
+  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where: includeIncompleteAds ? queryIncludeIncomplete : query,
+    where:
+      includeIncompleteAds || includeIncompleteAds == null
+        ? queryIncludeIncomplete
+        : query,
     limit: maxResults,
     order
   });
@@ -363,7 +348,5 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
 module.exports = {
   bulkUpsertRealEstates,
   getRealEstateById,
-  createRealEstate,
-  findRealEstatesForSearchRequest,
-  findRealEstateByAgencyId
+  findRealEstatesForSearchRequest
 };

app/helpers/db/searchRequest.js

@@ -3,6 +3,7 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -16,6 +17,22 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
+const findAllRequestsForCheckUp = async () => {
+  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
+  const checkupDate = new Date();
+  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
+
+  const dateQuery = {
+    notifiedAt: {
+      [Op.lte]: checkupDate
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: dateQuery
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -157,7 +174,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
+      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -176,7 +193,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -226,7 +246,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -275,7 +298,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -313,7 +339,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -347,7 +376,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -381,7 +413,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -423,10 +458,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query
+  //Tag to check if incomplete ads are accepted in query which is default
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.eq]: true
+      [Op.or]: {
+        [Op.eq]: true,
+        [Op.is]: null
+      }
     };
   }
 
@@ -438,5 +476,6 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate
+  findSearchRequestsForRealEstate,
+  findAllRequestsForCheckUp
 };

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -43,42 +42,6 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
-const findAllRequestsForCheckUp = async () => {
-  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate CHECK_UP_DAYS days from now
-  const date = new Date();
-  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
-  date.setDate(checkUpDate);
-  const dateQuery = {
-    createdAt: {
-      [Op.gte]: date
-    }
-  };
-
-  const excludedMatches = await db.SearchRequestMatch.findAll({
-    attributes: ["searchRequestId"],
-    where: dateQuery,
-    order: [["searchRequestId", "ASC"]]
-  });
-
-  const excludedRequestsAll = excludedMatches.map(match => {
-    return match.dataValues.searchRequestId;
-  });
-  //Removing duplicate search request id-s for optimization
-  const excludedRequests = [...new Set(excludedRequestsAll)];
-
-  const query = {
-    subscribed: true,
-    id: {
-      [Op.notIn]: excludedRequests
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: query
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -89,6 +52,5 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches,
-  findAllRequestsForCheckUp
+  findNotNotifiedMatches
 };

app/helpers/fetchWrapper.js

@@ -0,0 +1,24 @@
+const nodeFetch = require("node-fetch");
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
+} = require("../config/appConfig");
+
+const fetch = async (url, options = {}) => {
+  const newOptions = Object.assign({}, options);
+  if (!newOptions["headers"]) {
+    newOptions["headers"] = {};
+  }
+  newOptions["headers"]["User-Agent"] = USER_AGENT;
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    : url;
+
+  //
+  // console.log("Url for scraping:", urlAdaptedForScraping);
+
+  return nodeFetch(urlAdaptedForScraping, newOptions);
+};
+
+module.exports = fetch;

app/helpers/url.js

@@ -1,7 +1,4 @@
 const { getSearchRequest } = require("./db/searchRequest");
-const { getRealEstateById } = require("./db/realEstate");
-const { getKiviOriginalById } = require("./db/kiviOriginal");
-const validator = require("validator");
 
 const currentSearchRequest = async req => {
   const searchRequestId =
@@ -10,23 +7,6 @@ const currentSearchRequest = async req => {
 
   return await getSearchRequest(searchRequestId);
 };
-
-const currentRealEstate = async req => {
-  const realEstateId = req && req.params ? req.params["realEstateId"] : null;
-  if (!realEstateId) return null;
-
-  return await getRealEstateById(parseInt(realEstateId));
-};
-const currentKiviRealEstate = async req => {
-  const kiviRealEstateId =
-    req && req.params ? req.params["kiviRealEstateId"] : null;
-  if (!kiviRealEstateId || !validator.isUUID(kiviRealEstateId)) return null;
-
-  return await getKiviOriginalById(kiviRealEstateId);
-};
-
 module.exports = {
-  currentSearchRequest,
-  currentRealEstate,
-  currentKiviRealEstate
+  currentSearchRequest
 };

app/migrations/20200203114630-add-kiviOriginal-table.js

@@ -1,28 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    const tableFields = {
-      kiviAdId: {
-        type: Sequelize.UUID,
-        defaultValue: Sequelize.UUIDV4,
-        allowNull: false,
-        primaryKey: true
-      },
-      email: Sequelize.TEXT,
-      createdAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      },
-      updatedAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      }
-    };
-    return queryInterface.createTable("KiviOriginal", tableFields);
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable("KiviOriginal", {});
-  }
-};

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
+      type: Sequelize.DATE,
+      defaultValue: new Date()
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
+  }
+};

app/migrations/20200312210336-add-table-kivi-photos.js

@@ -1,39 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    const tableFields = {
-      id: {
-        type: Sequelize.BIGINT,
-        autoIncrement: true,
-        primaryKey: true,
-        allowNull: false
-      },
-      kiviAdId: {
-        type: Sequelize.UUID,
-        allowNull: false,
-        references: {
-          model: "KiviOriginal",
-          key: "kiviAdId"
-        }
-      },
-      photoUrl: {
-        type: Sequelize.TEXT,
-        allowNull: false
-      },
-      createdAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      },
-      updatedAt: {
-        type: Sequelize.DATE,
-        defaultValue: Sequelize.literal("NOW()")
-      }
-    };
-    return queryInterface.createTable("KiviOriginalAdsPhotos", tableFields);
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable("KiviOriginalAdsPhotos", {});
-  }
-};

app/models/kiviOriginal.js

@@ -1,21 +0,0 @@
-"use strict";
-
-module.exports = (sequalize, DataTypes) => {
-  const KiviOriginal = sequalize.define(
-    "KiviOriginal",
-    {
-      kiviAdId: {
-        type: DataTypes.UUID,
-        defaultValue: DataTypes.UUIDV4,
-        allowNull: false,
-        primaryKey: true
-      },
-      email: DataTypes.TEXT
-    },
-    {
-      freezeTableName: true
-    }
-  );
-
-  return KiviOriginal;
-};

app/models/kiviOriginalAdsPhotos.js

@@ -1,41 +0,0 @@
-"use strict";
-
-module.exports = (sequalize, DataTypes) => {
-  const KiviOriginalAdsPhotos = sequalize.define(
-    "KiviOriginalAdsPhotos",
-    {
-      id: {
-        type: DataTypes.BIGINT,
-        autoIncrement: true,
-        primaryKey: true,
-        allowNull: false
-      },
-      kiviAdId: {
-        type: DataTypes.UUID,
-        allowNull: false,
-        references: {
-          model: "KiviOriginal",
-          key: "kiviAdId"
-        }
-      },
-      photoUrl: {
-        type: DataTypes.TEXT,
-        allowNull: false
-      }
-    },
-    {
-      freezeTableName: true
-    }
-  );
-
-  KiviOriginalAdsPhotos.associate = models => {
-    KiviOriginalAdsPhotos.hasMany(models.KiviOriginal, {
-      foreignKey: "kiviAdId",
-      sourceKey: "kiviAdId",
-      targetKey: "kiviAdId",
-      as: "kiviOriginal"
-    });
-  };
-
-  return KiviOriginalAdsPhotos;
-};

app/models/searchRequest.js

@@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [
-          [
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0]
-          ]
-        ],
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -90,7 +82,11 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT
+    heatingType: DataTypes.TEXT,
+    notifiedAt: {
+      type: DataTypes.DATE,
+      defaultValue: new Date()
+    }
   });
 
   return SearchRequest;

app/public/main.css

@@ -158,47 +158,3 @@ h3 {
 .estates-link {
   color: rgba(0, 0, 0, 0.87);
 }
-
-.error {
-  color: #cc0033;
-}
-
-.custom-col {
-  margin-left: auto;
-  left: auto;
-  right: auto;
-}
-
-.dont-break-out {
-  overflow-wrap: break-word;
-  word-wrap: break-word;
-  -ms-word-break: break-all;
-  word-break: break-all;
-  word-break: break-word;
-  -ms-hyphens: auto;
-  -moz-hyphens: auto;
-  -webkit-hyphens: auto;
-  hyphens: auto;
-}
-
-.flex-direction-nav li a {
-  height: 50px;
-}
-
-.slider .slides li {
-  opacity: 1;
-  position: relative;
-}
-.dropzone {
-  background: white;
-  border-radius: 10px;
-  border: 4px dashed #02adba;
-  border-image: none;
-  max-width: 80%;
-  margin-left: auto;
-  margin-right: auto;
-}
-
-.dz-progress {
-  display: none;
-}

app/routes/index.js

@@ -7,21 +7,11 @@ const {
   getRealEstateTypes,
   postRealEstateTypes
 } = require("../controllers/realEstateTypes");
-const {
-  getPublishTypes,
-  postPublishTypes
-} = require("../controllers/publishRealEstateTypes");
-const {
-  getPublishInputs,
-  postPublishInputs
-} = require("../controllers/publishRealEstate");
-const { getViewRealEstate } = require("../controllers/viewRealEstate");
 const {
   getQueryReview,
   postQueryReview
 } = require("../controllers/queryReview");
 const { getGoAgain } = require("../controllers/goAgain");
-const { publishSuccess } = require("../controllers/publishSuccess");
 const { getLocation, postLocation } = require("../controllers/location");
 const { getUnsubscribe } = require("../controllers/unsubscribe");
 const { getRealEstates } = require("../controllers/realEstates");
@@ -38,16 +28,6 @@ router.get("/vrstanekretnine", getRealEstateTypes);
 router.post("/vrstanekretnine/:searchRequestId", postRealEstateTypes);
 router.post("/vrstanekretnine", postRealEstateTypes);
 
-router.get("/objavinekretninu/:kiviRealEstateId", getPublishTypes);
-router.get("/objavinekretninu", getPublishTypes);
-router.post("/objavinekretninu/:kiviRealEstateId", postPublishTypes);
-router.post("/objavinekretninu", postPublishTypes);
-
-router.get("/podacionekretnini/:kiviRealEstateId", getPublishInputs);
-router.post("/podacionekretnini/:kiviRealEstateId", postPublishInputs);
-
-router.get("/preglednekretnine/:realEstateId", getViewRealEstate);
-
 router.get("/lokacija/:searchRequestId", getLocation);
 router.post("/lokacija/:searchRequestId", postLocation);
 
@@ -61,8 +41,6 @@ router.get("/odjava/:searchRequestId", getUnsubscribe);
 
 router.get("/ponovo", getGoAgain);
 
-router.get("/uspjesnaobjava", publishSuccess);
-
 router.get("/nekretnine/:searchRequestId", getRealEstates);
 
 router.get("/redirect/:id", getRedirect);

app/services/notificationService.js

@@ -15,9 +15,10 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
-  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
+
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -35,7 +36,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-
+  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -76,6 +77,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
+
+        //Change time of notified At for searchReq
+        searchRequest.notifiedAt = new Date();
+        searchRequest.save();
       }
     }
   }
@@ -131,7 +136,7 @@ const notifyRequestsWithDailyOption = async () => {
 };
 
 const checkUpNotify = async () => {
-  const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
 
   const asyncSendEmailActions = [];
 
@@ -143,8 +148,12 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+
+    //Change time of notified At for searchReq
+    searchRequest.notifiedAt = new Date();
+    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions);
+  await Promise.all(asyncSendEmailActions);*/
 };
 
 module.exports = {

app/views/advancedFilters.ejs

@@ -61,9 +61,8 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
-      <% if (includeIncompleteAds) { %>
            checked
-      <% } %>>
+      >
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

app/views/layout.ejs

@@ -9,28 +9,16 @@
     gtag('js', new Date());
     gtag('config', '<%= process.env.GA_ID %>');
   </script>
- 
+
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.7.0/dropzone.css">
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no"/>
 
   <script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
-  <script src="//cdnjs.cloudflare.com/ajax/libs/validate.js/0.13.1/validate.min.js"></script>
-
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.7.0/dropzone.js"></script>
-  <script type="text/javascript">
-    Dropzone.autoDiscover = false;  
-  </script>
-
   <meta charset="UTF-8" />
   <link rel="stylesheet" href="/assets/main.css">
   <link rel="stylesheet" href="/assets/segment.css">
- <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/flexslider/2.7.2/flexslider.css" type="text/css" media="screen" /> 
-
-  <script defer src="https://cdnjs.cloudflare.com/ajax/libs/flexslider/2.7.2/jquery.flexslider.js"></script>
-  
 
   <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
   <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
@@ -59,9 +47,6 @@
     <% } else { %>
     <title>Kivi.ba</title>
   <% } %>
-
-  
-
 </head>
 
 <body>

app/views/location.ejs

@@ -217,7 +217,7 @@
   });
 </script>
 <script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
+  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.API_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
   async
   defer
 ></script>

app/views/publishAdditionalData.ejs

@@ -1,50 +0,0 @@
-<br>
-<div class="row">
-<% for (const input of additionalInputInputs){ %>
-    <div class="input-field col s12">
-    <textarea
-            id="<%= input.dbField %>"
-            form="publishForm"
-            name="<%= input.dbField %>"
-            cols="80" rows="15"
-            value="<%= additionalInputValues[input.dbField] !== undefined ? additionalInputValues[input.dbField] : ""%>"
-    ></textarea>
-    <label for="<%= input.dbField %>"><%= input.title %></label>
-</div>
-<% } %>
-</div>
-
-<br>
-<div class="row">
-
-<% for (const input of additionalBooleanPublishInputs){ %>
-  <p class="col s6 m4 l4">
-    <label class="checkbox-label">
-      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
-        <% if (additionalBooleanPublishValues[input.dbField]) { %>
-          checked
-        <% } %>>
-      <span><%= input.title %></span>
-    </label>
-  </p>
-<% } %>
-</div>
-<br>
-<% for (const input of additionalSegmentSelectInputs){ %>
-  <div>
-    <label class="checkbox-label"><%= input.title %>: </label><br><br>
-    <span class="segmented small">
-      <% for (const segmentObject of input.values) { %>
-        <% if (segmentObject.id!=="ANY") { %>
-        <label>
-            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
-            <% if (additionalSegmentSelectValues[input.dbField] === segmentObject.id) { %>
-               checked
-            <% } %>>
-            <span class="label"><%= segmentObject.title %></span>
-        </label>
-        <% } %>
-      <% } %>
-    </span>
-  </div>
-<% } %>

app/views/publishBasicData.ejs

@@ -1,49 +0,0 @@
-<br>
-<div class="row" id="basic-inputs">
-<% for (const input of basicInputInputs){ %>
-    <div class="input-field col s10 m5 l4">
-    <input
-            id="<%= input.dbField %>"
-            name="<%= input.dbField %>"
-            type="text"
-            value="<%= basicInputValues[input.dbField] !== undefined ? basicInputValues[input.dbField] : ""%>"
-    >
-    <label for="<%= input.dbField %>"><%= input.title %></label>
-</div>
-<% } %>
-</div>
-<br>
-<div class="row">
-<% for (const input of basicBooleanPublishInputs){ %>
-  <p>
-    <label class="checkbox-label">
-      <input type="checkbox" class="filled-in" name="<%= input.dbField %>"
-        <% if (basicBooleanPublishValues[input.dbField]) { %>
-          checked
-        <% } %>>
-      <span><%= input.title %></span>
-    </label>
-  </p>
-<% } %>
-</div>
-
-<br>
-<% for (const input of basicSegmentSelectInputs){ %>
-  <div>
-    <label class="checkbox-label"><%= input.title %>: </label><br><br>
-    <span class="segmented small">
-      <% for (const segmentObject of input.values) { %>
-        <label>
-            <input type="radio" name="<%= input.dbField %>" value="<%= segmentObject.id %>"
-            <% if (basicSegmentSelectValues[input.dbField] === segmentObject.id) { %>
-               checked
-            <% } %>>
-            <span class="label"><%= segmentObject.title %></span>
-        </label>
-      <% } %>
-    </span>
-  </div>
-<% } %>
-
-
-  

app/views/publishEnd.ejs

@@ -1,24 +0,0 @@
-<br>
-<div class="row center-align">
-    <h3>Vaš oglas je spreman!</h3> 
-    Unesite kontakt email i objavite oglas.
-
-<br>
-<div class="row center-align input-field col s3 m4 l5 form-group">
-  <input
-          id="email"
-          name="email"
-          type="email"
-  >
-  <div class="messages"></div>
-  <label for="email">Email</label>
-</div>
-</div>
-<br>
-<div class="row center-align">
-    <div class="col s6 push-s3">
-      <a id="submit" href="#" form="publishForm" class="welcome-center-button waves-effect waves-light btn">Objavi oglas</a>
-    </div>
-  </div>
-
-<br>

app/views/publishLocation.ejs

@@ -1,192 +0,0 @@
-<div class="row center-align">
-    <h3>
-        Izaberite lokaciju nekretnine na mapi. 
-    </h3>
-  </div>
-  
-  <div class="row center-align">
-    <div class="col s12 m12 l12 xl12">
-      <input
-        id="autocompleteInput"
-        placeholder="Unesite grad, naselje ili ulicu..."
-        type="text"
-      />
-    </div>
-  </div>
-  
-  <div class="row center-align">
-    <div class="col s12">
-      <div id="map"></div>
-    </div>
-  </div>
-  
-    <input type="hidden" name="north" id="north" />
-    <input type="hidden" name="south" id="south" />
-    <input type="hidden" name="east" id="east" />
-    <input type="hidden" name="west" id="west" />
-    <input type="hidden" name="lat" id="lat" />
-    <input type="hidden" name="lng" id="lng" />
-
-    <input type="hidden" name="locationInput" id="locationInput" />
-    <input type="hidden" name="locationInputData" id="locationInputData" />
-  
-  <script>
-    let autocomplete;
-    let map;
-    let places;
-    let geocoder;
-    let marker =false; //Initialy no marker on map
-  
-    function locateMe() {
-      if (navigator.geolocation) {
-        function onLocationSuccess(position) {
-          const coordinates =
-            position && position.coords ? position.coords : null;
-          if (coordinates) {
-            const longitude = coordinates.longitude || null;
-            const latitude = coordinates.latitude || null;
-  
-            if (longitude && latitude && map) {
-              map.setCenter({ lat: latitude, lng: longitude });
-              map.setZoom(16);
-            }
-          }
-        }
-  
-        navigator.geolocation.getCurrentPosition(onLocationSuccess);
-      }
-    }
-  
-    function initMap() {
-      const BOSNIA_BOUNDS = {
-        north: 45.7,
-        south: 41.69,
-        west: 15.55,
-        east: 20.77
-      };
-      const SARAJEVO_COORDINATES = {
-        lat: 43.85,
-        lng: 18.41
-      };
-  
-      const mapElement = document.getElementById("map");
-      const restrictMapPanningToBosniaOnly = {
-        latLngBounds: BOSNIA_BOUNDS,
-        strictBounds: true
-      };
-      const initialMapParams = {
-        center: SARAJEVO_COORDINATES,
-        zoom: 12,
-        restriction: restrictMapPanningToBosniaOnly,
-        mapTypeControl: false,
-        panControl: false,
-        zoomControl: true,
-        streetViewControl: false
-      };
-      map = new google.maps.Map(mapElement, initialMapParams);
-  
-      const inputElement = document.getElementById("autocompleteInput");
-      const restrictAutocompleteResultsToBosniaOnly = { country: "ba" };
-      const initialAutocompleteParams = {
-        types: ["geocode"],
-        componentRestrictions: restrictAutocompleteResultsToBosniaOnly,
-        fields: ["geometry", "types", "address_components"]
-      };
-  
-      autocomplete = new google.maps.places.Autocomplete(
-        inputElement,
-        initialAutocompleteParams
-      );
-      autocomplete.bindTo("bounds", map);
-      autocomplete.addListener("place_changed", onPlaceChanged);
-      pacSelectFirst(inputElement);
-      addLocateMeButton(map);
-      //Add event listener to position marker on map
-      google.maps.event.addListener(map, 'click', positionMarker);
-      
-    }
-
-    function positionMarker(event) {
-        let clickedLocation = event.latLng;
-        if(marker === false){
-            marker = new google.maps.Marker({
-                position: clickedLocation,
-                map: map,
-                draggable: true 
-            });
-            google.maps.event.addListener(marker, 'dragend', function(event){
-                markerLocation();
-            });
-        } else{
-            marker.setPosition(clickedLocation);
-        }
-    }
-
-    function addLocateMeButton(map) {
-      var parent = document.createElement("div");
-      parent.className = "locate-me-container";
-  
-      var a = document.createElement("a");
-      a.id = "locateMe";
-      a.className = "btn-floating";
-  
-      var i = document.createElement("i");
-      i.innerText = "gps_fixed";
-      i.className = "material-icons right";
-  
-      a.appendChild(i);
-      a.addEventListener("click", locateMe);
-      parent.appendChild(a);
-  
-      map.controls[google.maps.ControlPosition.RIGHT_BOTTOM].push(parent);
-    }
-  
-    function onPlaceChanged() {
-      const place = autocomplete.getPlace();
-      if (place.geometry) {
-        map.fitBounds(place.geometry.viewport);
-        map.setZoom(map.getZoom() + 1);
-        $("#locationInputData").val(JSON.stringify(place));
-      }
-    }
-  
-    function pacSelectFirst(input) {
-      // store the original event binding function
-      const _addEventListener = input.addEventListener
-        ? input.addEventListener
-        : input.attachEvent;
-  
-      function addEventListenerWrapper(type, listener) {
-        // Simulate a 'down arrow' keypress on hitting 'return' when no pac suggestion is selected,
-        // and then trigger the original listener.
-        if (type == "keydown") {
-          const originalListener = listener;
-          listener = function(event) {
-            const suggestionSelected = $(".pac-item-selected").length > 0;
-            if (event.key == "Enter" && !suggestionSelected) {
-              const simulatedDownArrow = $.Event("keydown", {
-                keyCode: 40,
-                which: 40
-              });
-              originalListener.apply(input, [simulatedDownArrow]);
-            }
-  
-            originalListener.apply(input, [event]);
-          };
-        }
-  
-        _addEventListener.apply(input, [type, listener]);
-      }
-  
-      input.addEventListener = addEventListenerWrapper;
-      input.attachEvent = addEventListenerWrapper;
-    }
-      
-  
-  </script>
-  <script
-    src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
-    async
-    defer
-  ></script>
-  

app/views/publishPhotos.ejs

@@ -1,12 +0,0 @@
-<br>
-
-<div action="/photos-upload" class="dropzone" id="photos-upload">
-      <div class="fallback">
-        <input name="file" type="file" multiple />
-      </div>
-      
-    </div>
-
-<br>
-
-<input type="hidden" name="imageUrls" id="imageUrls" value="">

app/views/publishRealEstate.ejs

@@ -1,290 +0,0 @@
-<br>
-<form id="publishForm" method="POST" novalidate >
-  <div class="row">
-    <div class="col s12">
-      <ul class="tabs">
-        <li class="tab col s3"><a href="#publishBasicData">Osnovni podaci</a></li>
-        <li class="tab col s3"><a href="#publishAdditionalData">Dodatni podaci</a></li>
-        <li class="tab col s2"><a href="#publishLocation">Lokacija</a></li>
-        <li class="tab col s2"><a href="#publishPhotos">Fotografije</a></li>
-        <li class="tab col s2"><a href="#publishEnd">Kraj</a></li>
-
-    </ul>
-    </div>
-    <div id="publishBasicData" class="col s12">
-      <%- include("./publishBasicData.ejs") %>
-    </div>
-    <div id="publishAdditionalData" class="col s12">
-      <%- include("./publishAdditionalData.ejs") %>
-    </div>
-    <div id="publishLocation" class="col s12">
-     <%- include("./publishLocation.ejs") %>
-    </div>
-    <div id="publishPhotos" class="col s12">
-      <%- include("./publishPhotos.ejs") %>
-     </div>
-    <div id="publishEnd" class="col s12">
-     <%- include("./publishEnd.ejs") %>
-    </div>
-  </div>
-
-  
-</form>
-
-<script>
-  function uuidv4() {
-      return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-        const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
-        return v.toString(16);
-      });
-    }
-
-    function getFileName(fileName) {
-        const encodedFileName = (uuidv4() + fileName).replace(/\s+/g, '');
-        return encodedFileName;
-    }
-
-    function upload() {
-      var file = $('#selector')[0].files[0];
-      uploadFile(file)
-    }
-
-
-    async function generateSignedURL(file) {
-      const fileName = getFileName(file.name);
-      const response = await fetch('/generateSignedURL?filename=' + fileName);
-      if (!response.ok) {
-        throw new Error('Network response for fetch was not ok.');
-      }
-      let signedUrl = await response.text();
-      signedUrl = signedUrl.replace(/\"/g, "")
-      await uploadFile(file, fileName, signedUrl);
-      return fileName;
-    }
-
-       function uploadFile(file, fileName, url) {
-    return  fetch(url, {
-        method: 'PUT',
-        headers: new Headers({'content-type': 'image/*'}),
-        mode: 'cors',
-        body: file
-      })
-        .then(response => response.text())
-        .then (response => {
-          return response;
-        }
-        )
-        .catch(error => $("#status").html(error)
-        )
-        .then(response => {
-          $("#imageUrls").val($("#imageUrls").val()+ fileName+"|");
-        });
-
-    }
-
-  $(document).ready(function(){
-    $('.tabs').tabs();
-
-    // Manual dropzone init
-    const dropzoneOptions = {
-					url: "/photos-upload", //can be a function that returns url ?
-          autoProcessQueue:false, //not to upload files automaticly
-          method: "put", //or post
-          parallelUploads: 1,
-          uploadMultiple: false, 
-					addRemoveLinks: true,
-					maxFilesize: 2, //MB,
-          resizeWidth: 600, 
-          maxFiles: 10, 
-          acceptedFiles: "image/*",
-        	dictDefaultMessage: `<span class="text-center">
-                              <h3>Prevuci fotografije ili klikni za dodavanje!</h3>
-                              (Maksimalno 10 fotografija.)
-                              </span>`,
-					dictResponseError: 'Error uploading file!',
-          dictRemoveFile: 'Izbriši ',
-          dictFileTooBig: 'Fajl je prevelik!',
-          dictInvalidFileType: 'Iabrani fajl nije fotografija!',
-          dictMaxFilesExceeded: 'Dostigli ste maksimalan broj fotografija!'
-				};
-      var photosUploader = new Dropzone('#photos-upload', dropzoneOptions);
-
-
-    //VALIDATION - WiP
-    //Helper validation functions
-    const isValidEmail = $email => {
-      const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
-      return $email && $email.length < 250 && simpleEmailRegex.test($email);
-      };
-
-    const isPresent = $input => {
-      return $input && $input!=="" && $input != null;
-      };
-
-    const isNumber = $input => {
-      const simpleNumberRegex = /([0-9]+[.|,][0-9])|([0-9][.|,][0-9]+)|([0-9]+)/;
-      return $input && $input.length <250 && simpleNumberRegex.test($input) && !isNaN($input);
-      };
-
-    const isInteger = $input => {
-      const simpleIntegerRegex = /^([+-]?[1-9]\d*|0)$/;
-      return $input && $input.length <250 && simpleIntegerRegex.test($input);
-      };
-
-    const form = document.querySelector("#publishForm");
-
-    function showErrors(form, errors) {
-    // We loop through all the inputs and show the errors for that input
-    //_.each(form.querySelectorAll("input[name], select[name]"), function(input) {
-      // Since the errors can be null if no errors were found we need to handle
-      // that
-      showErrorsForInput(input, errors && errors[input.name]);
-   // });
-  }
-
-  // Shows the errors for a specific input
-  function showErrorsForInput(input, errors) {
-    // This is the root of the input
-    var formGroup = closestParent(input.parentNode, "form-group"),
-      // Find where the error messages will be insert into
-      messages = formGroup.querySelector(".messages");
-    // First we remove any old messages and resets the classes
-    resetFormGroup(formGroup);
-    // If we have errors
-    if (errors) {
-      // we first mark the group has having errors
-      formGroup.classList.add("has-error");
-      // then we append all the errors
-      $.each(errors, function(error) {
-        addError(messages, errors[error]);
-      });
-
-    } else {
-      // otherwise we simply mark it as success
-      formGroup.classList.add("has-success");
-    }
-  }
-
-  // Recusively finds the closest parent that has the specified class
-  function closestParent(child, className) {
-    if (!child || child == document) {
-      return null;
-    }
-    if (child.classList.contains(className)) {
-      return child;
-    } else {
-      return closestParent(child.parentNode, className);
-    }
-  }
-
-  function resetFormGroup(formGroup) {
-    formGroup.classList.remove("has-error");
-    formGroup.classList.remove("has-success");
-    $.each(formGroup.querySelectorAll(".help-block.error"), function(el) {
-      el.parentNode.removeChild(el);
-    });
-  }
-
-  // Adds the specified error with the following markup
-  // <p class="help-block error">[message]</p>
-  function addError(messages, error) {
-    var block = document.createElement("p");
-    block.classList.add("help-block");
-    block.classList.add("error");
-    block.innerText = error;
-    messages.appendChild(block);
-  }
-
-  const validate = (input) => {
-
-    let valid=true;
-    let errorMsg =[];
-    let constraint = input.constraint[0];
-
-    switch (constraint) {
-      case "required":
-      valid = isPresent ($(`#${input.dbField}`).val());
-      errorMsg = ["Ovo je obavezno polje."];
-      break;
-      case "numerical":
-      valid = isNumber ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite brojcanu vrijednost."];
-      break;
-      case "integer":
-      valid = isInteger ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite cijeli broj."];
-      break;
-      default :
-      valid = true;
-     }
-    if (!valid) {
-    const inputField = document.querySelector(`#${input.dbField}`);
-    showErrorsForInput( inputField, errorMsg);
-    return false;
-    } else {
-    return true;
-    }
-
-}
-
-    $("#submit").click( async function (e) {
-      e.preventDefault();
-
-      if (marker) {
-        const currentLocation = marker.getPosition();
-
-        $("#lat").val(currentLocation.lat());
-
-        $("#lng").val(currentLocation.lng());
-
-        $("#locationInput").val(
-          document.getElementById("autocompleteInput").value
-          );
-      } else {
-        $("#lat").val(0);
-
-        $("#lng").val(0);
-      }
-      
-       //Tag for checking of error presence 
-       let hasErrors = false;
-      //Check if email is valid
-       const validEmail = isValidEmail($("#email").val());
-       //Show messeges for invalid email is present
-      if (!validEmail) {
-        const errorMsgs = ["Unesite validan email."];
-        const email = document.querySelector("#email");
-        showErrorsForInput( email, errorMsgs) 
-        hasErrors = true;
-      };
-      //Check if other input fields are valid - vratiti se na ovo!!
-      //const basicInputInputs= document.getElementById("basic-inputs").getElementsByTagName("input");
-      
-      //alert(JSON.stringify(""));
-      /*
-      $.each(basicInputInputs, function (input) {
-        alert(input);
-        validate (input);
-      })
-      for (const input of  basicInputInputs ) {
-        alert(input.getAttribute(name));
-
-        validate (input);
-      } */
-      const addedFiles = photosUploader.files.filter(file => file.status!=="error");
-      const asyncUpload =[];
-
-        addedFiles.forEach( file => {
-          asyncUpload.push(generateSignedURL(file)); 
-        })
-     
-        if (!hasErrors) {
-            await Promise.all(asyncUpload);
-            $("#publishForm").submit();
-
-      };
-  
-    });
-  });
- 
-</script>

app/views/publishSuccess.ejs

@@ -1,11 +0,0 @@
-<br>
-<div class="row center-align">
-    <p>Vaš oglas je spašen u Kivi bazu.</p>
-    <br>
-    <div class="row center-align">
-        <img src="../assets/images/logo.svg" alt="kivi logo" width="160">
-    </div>
-    <br>
-    <p>Poslali smo potvrdni email sa detaljima oglasa na Vašu email adresu.</p>
-    <a href="/" class="">Nova pretraga</a>
-</div>

app/views/realEstateType.ejs

@@ -9,7 +9,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= labelAdType[0] %></span>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
                 </label>
 
                 <label>
@@ -17,7 +17,7 @@
                     <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
                     checked
                     <% } %>>
-                    <span class="label"><%= labelAdType[1] %></span>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
                 </label>
 		    </span>
         </div>

app/views/viewRealEstate.ejs

@@ -1,129 +0,0 @@
-<br/>
-<div class="row col s12 center-align">
-  <div class="col s6 center-align distinguished">
-    <div><%= showAdType %> </div>
-  </div>
-  <div class="col s6 center-align distinguished">
-    <%= showRealEstateType %> 
-  </div>
-</div>
-
- <section class="slider">
-  <div class="flexslider" >
-    <ul class="slides">
-      <% for (const photoUrl of realEstatePhotosUrls) { %>
-      <li class="flex-li">
-        <img src=<%= photoUrl %> alt=""/>
-    </li>
-    <% } %>
-    </ul>
-  </div>
-</section> 
-
-<br/>
-<br>
-<div class="row col s12">
-<% for (const field of inputFields){ %>
-  <p>
-      <span class="col s4"><%= field.title %></span>
-      <span class="col s8 distinguished dont-break-out"><%= allInputValues[field.dbField] %></span>
-  </p>
-  <br>
-<% } %>
-</div>
-
-<br>
-<div class="row">
-<% for (const field of segmentFields){ %>
-  <p>
-      <span class="col s4"><%= field.title.replace(/>/g,'') %></span>
-      <% for (const segmentObject of field.values) { %>
-        <% if (allSegmentSelectedValues[field.dbField] === segmentObject.id) { %>
-                <span class="col s8 distinguished"><%= segmentObject.title.replace(/>/g,'') %></span>
-            <% } %>
-      <% } %>
-  </p>
-  <br>
-<% } %>
-</div>
-<br>
-<div class="row col s12">
-<% for (const field of booleanFields){ %>
-  <p class="col s4">
-    <span>&#10003;</span>
-      <span><%= field.title %></span>
-    
-  </p>
-<% } %>
-</div>
-
-<div class="row center-align ">
-  <div class="distinguished"> 
-    <span>Lokacija nekretnine</span>
-    
-  </div>
-  <br>
-    <br>
-  <div class="col s12">
-    <div id="map"></div>
-  </div>
-</div>
-
-<script type="text/javascript">
-       $(window).load(function() {
-           $('.flexslider').flexslider({
-           animation: "slide",
-           smoothHeight: true
-          });
-        });
-</script>
-
-<script>
-  //Setting up image gallery - carousel
-  
-
-  //Setting up location map
-  let map;
-
-  function initMap() {
-    const BOSNIA_BOUNDS = {
-      north: 45.7,
-      south: 41.69,
-      west: 15.55,
-      east: 20.77
-    };
-    const ESTATE_COORDINATES = {
-      lat: <%= locationLat %>,
-      lng: <%= locationLong %>
-    };
-
-    const mapElement = document.getElementById("map");
-    const restrictMapPanningToBosniaOnly = {
-      latLngBounds: BOSNIA_BOUNDS,
-      strictBounds: true
-    };
-    const initialMapParams = {
-      center: ESTATE_COORDINATES,
-      zoom: 13,
-      restriction: restrictMapPanningToBosniaOnly,
-      mapTypeControl: false,
-      panControl: false,
-      zoomControl: true,
-      streetViewControl: false
-    };
-    map = new google.maps.Map(mapElement, initialMapParams);
-    
-    marker = new google.maps.Marker({
-                position: ESTATE_COORDINATES,
-                map: map,
-            });
-   
-  }
-
-
-</script>
-<script
-  src="https://maps.googleapis.com/maps/api/js?key=<%= process.env.GOOGLE_MAP_KEY %>&language=bs&libraries=places&callback=initMap"
-  async
-  defer
-></script>

app/views/welcome.ejs

@@ -18,20 +18,7 @@
     </div>
     <input type="hidden" id="adType" name="adType">
 </form>
-<div class="row center-align">
-    <div>Objavite svoj oglas.</div>
-</div>
-<form method="POST" name="welcomePublishForm">
-    <div class="row center-align">
-        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
-            <a href="#" onclick="publishSaleClick()" class="welcome-center-button btn">Prodaj</a>
-        </div>
-        <div class="col s5 m4 l3 push-s1 push-m2 push-l3">
-            <a href="#" onclick="publishRentClick()" class="welcome-center-button btn">Iznajmi</a>
-        </div>
-    </div>
-    <input type="hidden" id="publishAdType" name="publishAdType">
-</form>
+
 <script>
     function saleClick(){
         $("#adType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
@@ -42,13 +29,4 @@
         $("#adType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
         document.welcomeForm.submit();
     }
-    function publishSaleClick(){
-        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_SALE.id %>");
-        document.welcomePublishForm.submit();
-    }
-
-    function publishRentClick(){
-        $("#publishAdType").val("<%= AD_TYPE.AD_TYPE_RENT.id %>");
-        document.welcomePublishForm.submit();
-    }
 </script>

development.env

@@ -8,8 +8,9 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
-SETTINGS=Variable to denote development, staging and production
+ENVIRONMENT=Variable to denote development, staging and production
 
+USER_AGENT=User agent header to send in fetch requests
 
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
@@ -19,10 +20,12 @@ CHECK_UP_DAYS=Check up email is sent after this number of days without notificat
 GA_ID=Google Analytics ID
 
 #=============== GOOGLE MAPS =============#
-GOOGLE_MAP_KEY=(your-key-here)
+API_MAP_KEY=(your-key-here)
 
-#=============== GOOGLE STORAGE =============#
-GOOGLE_APPLICATION_CREDENTIALS="Path to json key file"
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api 
+NUMBER_OF_CONCURRENT_REQ_SCRAPER_API= Number of requests to send concurrently to Srcaper API proxy
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
@@ -34,6 +37,7 @@ SOURCE_EMAIL=info@saburly.com
 CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds
 STOP_CRAWLER=Non-zero value will skip crawler execution
 PRINT_CRAWLER_DEBUG_INFO=Non-zero value will print crawler debugging info to the server console
+
 #==OLX==
 OLX_MAX_PAGES=Restrict crawler to this number of pages
 OLX_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -42,6 +46,7 @@ OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be
 OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
 OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+
 #==RENTAL==
 RENTAL_MAX_PAGES=Restrict crawler to this number of pages
 RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -72,4 +77,4 @@ AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

help.js

@@ -1,237 +0,0 @@
-(function() {
-  // Before using it we must add the parse and format functions
-  // Here is a sample implementation using moment.js
-  validate.extend(validate.validators.datetime, {
-    // The value is guaranteed not to be null or undefined but otherwise it
-    // could be anything.
-    parse: function(value, options) {
-      return +moment.utc(value);
-    },
-    // Input is a unix timestamp
-    format: function(value, options) {
-      var format = options.dateOnly ? "YYYY-MM-DD" : "YYYY-MM-DD hh:mm:ss";
-      return moment.utc(value).format(format);
-    }
-  });
-
-  // These are the constraints used to validate the form
-  var constraints = {
-    email: {
-      // Email is required
-      presence: true,
-      // and must be an email (duh)
-      email: true
-    },
-    password: {
-      // Password is also required
-      presence: true,
-      // And must be at least 5 characters long
-      length: {
-        minimum: 5
-      }
-    },
-    "confirm-password": {
-      // You need to confirm your password
-      presence: true,
-      // and it needs to be equal to the other password
-      equality: {
-        attribute: "password",
-        message: "^The passwords does not match"
-      }
-    },
-    username: {
-      // You need to pick a username too
-      presence: true,
-      // And it must be between 3 and 20 characters long
-      length: {
-        minimum: 3,
-        maximum: 20
-      },
-      format: {
-        // We don't allow anything that a-z and 0-9
-        pattern: "[a-z0-9]+",
-        // but we don't care if the username is uppercase or lowercase
-        flags: "i",
-        message: "can only contain a-z and 0-9"
-      }
-    },
-    birthdate: {
-      // The user needs to give a birthday
-      presence: true,
-      // and must be born at least 18 years ago
-      date: {
-        latest: moment().subtract(18, "years"),
-        message: "^You must be at least 18 years old to use this service"
-      }
-    },
-    country: {
-      // You also need to input where you live
-      presence: true,
-      // And we restrict the countries supported to Sweden
-      inclusion: {
-        within: ["SE"],
-        // The ^ prevents the field name from being prepended to the error
-        message: "^Sorry, this service is for Sweden only"
-      }
-    },
-    zip: {
-      // Zip is optional but if specified it must be a 5 digit long number
-      format: {
-        pattern: "\\d{5}"
-      }
-    },
-    "number-of-children": {
-      presence: true,
-      // Number of children has to be an integer >= 0
-      numericality: {
-        onlyInteger: true,
-        greaterThanOrEqualTo: 0
-      }
-    }
-  };
-
-  // Hook up the form so we can prevent it from being posted
-  var form = document.querySelector("form#main");
-  form.addEventListener("submit", function(ev) {
-    ev.preventDefault();
-    handleFormSubmit(form);
-  });
-
-  // Hook up the inputs to validate on the fly
-  var inputs = document.querySelectorAll("input, textarea, select");
-  for (var i = 0; i < inputs.length; ++i) {
-    inputs.item(i).addEventListener("change", function(ev) {
-      var errors = validate(form, constraints) || {};
-      showErrorsForInput(this, errors[this.name]);
-    });
-  }
-
-  function handleFormSubmit(form, input) {
-    // validate the form against the constraints
-    var errors = validate(form, constraints);
-    // then we update the form to reflect the results
-    showErrors(form, errors || {});
-    if (!errors) {
-      showSuccess();
-    }
-  }
-
-  // Updates the inputs with the validation errors
-  function showErrors(form, errors) {
-    // We loop through all the inputs and show the errors for that input
-    _.each(form.querySelectorAll("input[name], select[name]"), function(input) {
-      // Since the errors can be null if no errors were found we need to handle
-      // that
-      showErrorsForInput(input, errors && errors[input.name]);
-    });
-  }
-
-  // Shows the errors for a specific input
-  function showErrorsForInput(input, errors) {
-    // This is the root of the input
-    var formGroup = closestParent(input.parentNode, "form-group"),
-      // Find where the error messages will be insert into
-      messages = formGroup.querySelector(".messages");
-    // First we remove any old messages and resets the classes
-    resetFormGroup(formGroup);
-    // If we have errors
-    if (errors) {
-      // we first mark the group has having errors
-      formGroup.classList.add("has-error");
-      // then we append all the errors
-      _.each(errors, function(error) {
-        addError(messages, error);
-      });
-    } else {
-      // otherwise we simply mark it as success
-      formGroup.classList.add("has-success");
-    }
-  }
-
-  // Recusively finds the closest parent that has the specified class
-  function closestParent(child, className) {
-    if (!child || child == document) {
-      return null;
-    }
-    if (child.classList.contains(className)) {
-      return child;
-    } else {
-      return closestParent(child.parentNode, className);
-    }
-  }
-
-  function resetFormGroup(formGroup) {
-    // Remove the success and error classes
-    formGroup.classList.remove("has-error");
-    formGroup.classList.remove("has-success");
-    // and remove any old messages
-    _.each(formGroup.querySelectorAll(".help-block.error"), function(el) {
-      el.parentNode.removeChild(el);
-    });
-  }
-
-  // Adds the specified error with the following markup
-  // <p class="help-block error">[message]</p>
-  function addError(messages, error) {
-    var block = document.createElement("p");
-    block.classList.add("help-block");
-    block.classList.add("error");
-    block.innerText = error;
-    messages.appendChild(block);
-  }
-
-  function showSuccess() {
-    // We made it \:D/
-    alert("Success!");
-  }
-})();
-/////////////////////////////////////////////////
-const isPresent = $input => {
-  return $input && $input!=="" && $input != null;
-}
-
-const isNumber = $input => {
-  const simpleNumberRegex = /[+-]?(?:\d*[.,])?\d+/;
-  return $input && $input.length <250 && simpleNumberRegex.test($input);
-
-}
-
-const isInteger = $input => {
-  const simpleIntegerRegex = /^([+-]?[1-9]\d*|0)$/;
-  return $input && $input.length <250 && simpleIntegerRegex.test($input);
-
-}
-
-const validate = (input) => {
-
-  const valid;
-  const errorMsg;
-  const constraint = input.constraint[0];
-
-  switch (constraint) {
-    case "required":
-      valid = isPresent ($(`#${input.dbField}`).val());
-      errorMsg = ["Ovo je obavezno polje."];
-      break;
-    case "numerical":
-      valid = isNumber ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite brojcanu vrijednost."];
-      break;
-    case "integer":
-      valid = isInteger ($(`#${input.dbField}`).val());
-      errorMsg = ["Unesite cijeli broj."];
-
-      break;
-      default :
-      valid = true;
-    }
-    if (!valid) {
-      const inputField = document.querySelector(`#${input.dbField}`);
-    showErrorsForInput( inputField, errorMsg);
-    return false;
-  } else {
-    return true;
-  }
-
-  }
-

help2.js

@@ -1,112 +0,0 @@
-const validatorFunction = () => {
-  // These are the constraints used to validate the form --just email for now!
-  const constraints = {
-    email: {
-      email: {
-        message: "Proba"
-      },
-      // Email is required
-      presence: true,
-      // and must be an email (duh)
-      email: true
-    }
-  };
-
-  // Hook up the inputs to validate on the fly
-  const inputs = document.querySelectorAll("#email");
-  // inputs.on("change", ev => {
-  //     const errors = validate(form, constraints) || {};
-  //     showErrorsForInput(this, errors[this.name]);
-  //   });
-  // var inputs = document.querySelectorAll("input, textarea, select");
-  for (var i = 0; i < inputs.length; ++i) {
-    inputs.item(i).addEventListener("change", function(ev) {
-      var errors = validate(form, constraints) || {};
-      showErrorsForInput(this, errors[this.name]);
-    });
-  }
-
-  const handleFormSubmit = (form, input) => {
-    // validate the form against the constraints
-    const errors = validate(form, constraints);
-    //
-    console.log("handleFormSubmit error:", errors);
-    // then we update the form to reflect the results
-    showErrors(form, errors || {});
-    if (!errors) {
-      showSuccess();
-    }
-  };
-
-  // Updates the inputs with the validation errors
-  const showErrors = (form, errors) => {
-    // We loop through all the inputs and show the errors for that input
-    $.each(form.querySelectorAll("input[name], select[name]"), input => {
-      // Since the errors can be null if no errors were found we need to handle
-      // that
-      showErrorsForInput(input, errors && errors[input.name]);
-    });
-    //showErrorsForInput(email, errors && errors[email]);
-  };
-
-  // Shows the errors for a specific input
-  const showErrorsForInput = (input, errors) => {
-    // This is the root of the input
-    const formGroup = closestParent(input.parentNode, "form-group"),
-      // Find where the error messages will be insert into
-      messages = formGroup.querySelector(".messages");
-    // First we remove any old messages and resets the classes
-    resetFormGroup(formGroup);
-    // If we have errors
-    if (errors) {
-      //
-      console.log("errors:", errors);
-      // we first mark the group has having errors
-      formGroup.classList.add("has-error");
-      // then we append all the errors
-      $.each(errors, error => {
-        addError(messages, errors[error]);
-      });
-    } else {
-      // otherwise we simply mark it as success
-      formGroup.classList.add("has-success");
-    }
-  };
-
-  // Recusively finds the closest parent that has the specified class
-  const closestParent = (child, className) => {
-    if (!child || child == document) {
-      return null;
-    }
-    if (child.classList.contains(className)) {
-      return child;
-    } else {
-      return closestParent(child.parentNode, className);
-    }
-  };
-
-  const resetFormGroup = formGroup => {
-    // Remove the success and error classes
-    formGroup.classList.remove("has-error");
-    formGroup.classList.remove("has-success");
-    // and remove any old messages
-    $.each(formGroup.querySelectorAll(".help-block.error"), el => {
-      el.parentNode.removeChild(el);
-    });
-  };
-
-  // Adds the specified error with the following markup
-  // <p class="help-block error">[message]</p>
-  const addError = (messages, error) => {
-    const block = document.createElement("p");
-    block.classList.add("help-block");
-    block.classList.add("error");
-    block.innerText = error;
-    messages.appendChild(block);
-  };
-
-  const showSuccess = () => {
-    // We made it \:D/
-    alert("Success!");
-  };
-};

index.js

@@ -5,10 +5,6 @@ const layout = require("express-layout");
 const compression = require("compression");
 const forceSSL = require("./app/helpers/forceSSL");
 
-const { Storage } = require("@google-cloud/storage");
-const validate = require("validate.js");
-const cors = require("cors");
-
 const {
   APP_PORT,
   CRAWLER_INTERVAL,
@@ -23,8 +19,6 @@ const {
 
 const app = express();
 
-app.use(cors());
-
 app.use(forceSSL());
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({ extended: true }));
@@ -56,36 +50,3 @@ const crawl = () => {
 setInterval(crawl, CRAWLER_INTERVAL * 1000);
 
 setInterval(checkUpNotify, 1000 * 60 * 60 * 24);
-
-//Google storage req
-const PROJECT_ID = "marketalarm";
-const KEY_FILENAME = ""; //relative path
-const BUCKET_NAME = "marketalarm-photos";
-const storage = new Storage();
-
-const bucket = storage.bucket(BUCKET_NAME);
-
-app.get("/generateSignedURL", (req, res) => {
-  async function generateSignedUrl() {
-    // console.log("Started server function!");
-
-    const options = {
-      //Tried to define Google ID and private key while debugging
-      version: "v2", //tried v4 also
-      action: "write",
-      contentType: "image/*", //tried without and with specific image/png ex.
-      expires: Date.now() + 86400000
-    };
-    const filename = req.query.filename;
-
-    // console.log("Filename: ", filename);
-    // console.log("Bucket name:", bucket.name);
-
-    const [url] = await bucket.file(filename).getSignedUrl(options);
-
-    //console.log(`The signed url is ${url}.`);
-
-    res.status(200).send(url);
-  }
-  generateSignedUrl().catch(console.error);
-});

marketalarm-cors.json

@@ -1,12 +0,0 @@
-[
-  {
-    "origin": ["*"],
-    "responseHeader": [
-      "Content-Type",
-      "Access-Control-Allow-Origin",
-      "x-goog-resumable"
-    ],
-    "method": ["GET", "HEAD", "DELETE", "POST", "PUT", "OPTIONS"],
-    "maxAgeSeconds": 3600
-  }
-]

marketalarm-d4b71394407e.json

@@ -1,12 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "marketalarm",
-  "private_key_id": "d4b71394407eb3dba9e431851dab60b198d6985d",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDTE6dkFr0bzDXg\n7ghMxkzq8cajqqqes9JZVqsXh+b/kFJYmEImFUILJJZdI080KM2sEYsIapBCxhMP\nFH017f/gfH3jnRbp3c70hghNh8noSTsq7kPA4l25o8GQnJ6AS+nhy8umPjb4KzX9\nkmC6OOD4P8mAmGqhoUv4s2jld1cXNur6NJjCpjEd2cH3SUbI71oA3V/4W8aK4dvS\n660kLY0PRt7mCiITe0hbTUBZY48W2ijZ7wM2r0HUtPG9XEeGMGmNsC+qD2oWxUU3\nvnm7l1fEIUvLYF4GrLRDJDSkpChBXNcWhoGV2AOvuTc+yghU2+lJWqrKcpLlI23E\nlVJjt9UhAgMBAAECggEABatr8sxq+SQOf9hSIe3Me9Kc1nunrC42scFHRKBNxahJ\ndXw5B9FQPh738Cqhk0xEz6hlrln1Agj6HhRIz8U0r9R+z4TRRr6kfnWmBZAMShu0\nC4JW448abpAYx8CQ/CvRmq2GlF+/M+QBeqpLS8gPzyaKTB/5IBaKG8Bn0fXXQZ2e\n7RaTpGx62jq79omPwiKz0PMVBGZrzPu8Z4tW47muV51osdKSNVgsXb4gCZl28zN2\n6zzY1ZK7u89MesY8joILMHm8cw0oyv9o+RVGEa1I1nq2q1A8ftZny4p7kUA/ITZX\nEZ6SCOP87z9HeVCr8lzexcovD8uZCOTYpcfotlSjGQKBgQD1VVGU2bzExiV0XQGc\n8n6m4TR6Y9zwXBiQPe0rXPZhvsj8QMTXk+L0ejCo4m7NF1dEyH6u+qX6wjNL1Hm9\nN/ZuDFqYtd9w6cQ8CtDZ6QZIE60k6tLQhMNRNMvHdMfedq+VOz3LX6TdyTnv8dP3\nbEsD8wIfFd6t5wNgeZkbKsNxBwKBgQDcQQsUppjglGpUoz7lGHKbFcKMPpIj9fMY\nfze1DXeTAtHGxGm2F10WZvxOEs4DCOUllBlarL5xDAJIJHk/NYlgnI6MJXMLro6d\nsb4iNTUuJKdqAijyOaZQUADJpdYKGwu5y66PUOuojWFV9kiamquXduJ9jzOa1vr1\nSJPUy2YGlwKBgQDNJrpgwa8z0QozAy89Ih68x+fNTMLNkAXOYKp6L3OsixCguDyi\nlP0dOSyFnUvQXutQDmS5R8oSJeElURk4HJsKrXP47WVak3DQUK8S+eSR0zpfe6os\nSkjWGFMriEE2i4MKRI7JCULhX8r+FfgNl9YnCEfG3M/oFhzhyO06JYlncwKBgF6n\nBSAGyEQbA+cDkI/bhcToAQdMDHmvxJyOb147P1vKJmSJG/TI7ZQnBd53blkXhYI0\ntwCko+LpCkH+iqyDUVpXbVsE7P/kMB3MuKzyuLvvvJJuAzK1W6e/+daukeEd5lge\nFBI68EsrFt1eTa1DMuKQkJzs4Xx1TrwCSKV2E45ZAoGBAIZkXyAOhwqCxwDF7B69\nt/7CWs0gPGqp6lFO7fgt7jPmcmSEr/xgUbBDFwd7D49jpXVgCEtr1Bd6MItlu/Ns\nXgXyOa5LPQmglF7UtnvuQLASBy5X6boKaf3sz7I5eho1kXczPGQUHfR5e0DaTND3\nTi2NLIAUci8T7hc8mONdeEHD\n-----END PRIVATE KEY-----\n",
-  "client_email": "marketalarm-photos-service-acc@marketalarm.iam.gserviceaccount.com",
-  "client_id": "115644068453290488813",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/marketalarm-photos-service-acc%40marketalarm.iam.gserviceaccount.com"
-}

package.json

@@ -17,7 +17,9 @@
     "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js",
-    "test-rental-scraper": "cd test && node rentalScrapeTest.js"
+    "test-saljic-scraper": "cd test && node saljicScrapeTest.js",
+    "test-rental-scraper": "cd test && node rentalScrapeTest.js",
+    "test-scraper-api": "cd test && node scraperAPITest.js"
   },
   "repository": {
     "type": "git",
@@ -30,13 +32,11 @@
   },
   "dependencies": {
     "2checkout-node": "0.0.1",
-    "@google-cloud/storage": "^4.5.0",
     "@sendgrid/mail": "^6.3.1",
     "aws-sdk": "^2.422.0",
     "bluebird": "^3.5.5",
     "cheerio": "^1.0.0-rc.2",
     "compression": "^1.7.4",
-    "cors": "^2.8.5",
     "dotenv": "^7.0.0",
     "ejs": "^2.6.1",
     "eslint-plugin-prettier": "^3.1.2",
@@ -52,10 +52,9 @@
     "pg": "^7.10.0",
     "prettier": "^1.19.1",
     "react-step-wizard": "^5.1.0",
+    "scraperapi-sdk": "^1.0.3",
     "sequelize": "^5.18.4",
-    "sequelize-cli": "^5.5.0",
-    "validate.js": "^0.13.1",
-    "validator": "^12.2.0"
+    "sequelize-cli": "^5.5.0"
   },
   "devDependencies": {
     "nodemon": "^1.19.0"

test/olxScrapeTest.js

@@ -9,7 +9,7 @@ if (urlToScrape) {
 
   (async () => {
     const data = await crawler.scrapeAd(urlToScrape);
-    console.log(data);
+    console.log("Scraped data:", data);
   })();
 } else {
   console.log("No URL to scrape. Use like this : ");

test/saljicScrapeTest.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const saljicCrawler = require("../app/crawler/specificCrawlers/saljic");
+
+const urlToScrape = process.argv[2] || undefined;
+
+if (urlToScrape) {
+  const crawler = new saljicCrawler();
+
+  (async () => {
+    const data = await crawler.scrapeAd(urlToScrape);
+    console.log("Scraped data:", data);
+  })();
+} else {
+  console.log("No URL to scrape. Use like this : ");
+  console.log("npm run test-saljic-scraper -- URL_TO_SCRAPE");
+}

test/scraperAPITest.js

@@ -0,0 +1,19 @@
+const { SCRAPER_API_KEY } = require("../app/config/appConfig");
+
+const scraperapiClient = require("scraperapi-sdk")(SCRAPER_API_KEY);
+
+async function logUsedConcurrentReq() {
+  try {
+    const response = await scraperapiClient.account();
+    const dateOfLog = new Date().toLocaleString();
+    console.log(
+      dateOfLog,
+      " Number of concurrent requests: ",
+      response.concurrentRequests
+    );
+  } catch (err) {
+    console.log(err.message);
+  }
+}
+
+setInterval(logUsedConcurrentReq, 1000);