replace arrow function with old style function

move email to different column on unsubscribe action

See merge request saburly/marketalarm/web!70

app/common/enums.js

@@ -1,12 +1,21 @@
-const PRICE_SLIDER_OPTIONS = {
+const PRICE_SLIDER_OPTIONS_SALE = {
   start: [50000, 85000],
   range: {
     min: [0],
     max: [300000]
   },
   step: 1000,
-  connect: true,
-  tooltips: true
+  connect: true
+};
+
+const PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 500],
+  range: {
+    min: [0],
+    max: [2000]
+  },
+  step: 50,
+  connect: true
 };
 
 //This will be used for Flats, Apartments, Houses
@@ -17,8 +26,7 @@ const HOME_SIZE_SLIDER_OPTIONS = {
     max: [400]
   },
   step: 5,
-  connect: true,
-  tooltips: true
+  connect: true
 };
 
 const GARDEN_SIZE_SLIDER_OPTIONS = {
@@ -28,8 +36,7 @@ const GARDEN_SIZE_SLIDER_OPTIONS = {
     max: [10000]
   },
   step: 100,
-  connect: true,
-  tooltips: true
+  connect: true
 };
 
 const LAND_SIZE_SLIDER_OPTIONS = {
@@ -39,8 +46,7 @@ const LAND_SIZE_SLIDER_OPTIONS = {
     max: [100000]
   },
   step: 100,
-  connect: true,
-  tooltips: true
+  connect: true
 };
 const GARAGE_SIZE_SLIDER_OPTIONS = {
   start: [10, 20],
@@ -49,8 +55,7 @@ const GARAGE_SIZE_SLIDER_OPTIONS = {
     max: [150]
   },
   step: 2,
-  connect: true,
-  tooltips: true
+  connect: true
 };
 
 const GARAGE_PRICE_SLIDER_OPTIONS = {
@@ -60,28 +65,45 @@ const GARAGE_PRICE_SLIDER_OPTIONS = {
     max: [100000]
   },
   step: 500,
-  connect: true,
-  tooltips: true
+  connect: true
 };
 
 const AD_TYPE = {
-  AD_TYPE_SALE: "SALE",
-  AD_TYPE_RENT: "RENT"
+  AD_TYPE_SALE: {
+    id: 1,
+    stringId: "SALE",
+    title: "Prodaja"
+  },
+  AD_TYPE_RENT: {
+    id: 2,
+    stringId: "RENT",
+    title: "Najam"
+  },
+  AD_TYPE_REQUEST: {
+    id: 3,
+    stringId: "REQUEST",
+    title: "Potražnja"
+  }
 };
 
 const AD_CATEGORY = {
+  ALL: {
+    id: "ALL"
+  },
   FLAT: {
     id: "FLAT",
     title: "Stan",
     hasGardenSize: false,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   HOUSE: {
     id: "HOUSE",
     title: "Kuća",
     hasGardenSize: true,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   },
@@ -89,35 +111,40 @@ const AD_CATEGORY = {
     id: "OFFICE",
     title: "Kancelarija",
     hasGardenSize: false,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   LAND: {
     id: "LAND",
     title: "Zemljište",
     hasGardenSize: false,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
   },
   APARTMENT: {
     id: "APARTMENT",
     title: "Apartman",
     hasGardenSize: false,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   GARAGE: {
     id: "GARAGE",
     title: "Garaža",
     hasGardenSize: false,
-    priceSliderOptions: GARAGE_PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: GARAGE_SIZE_SLIDER_OPTIONS
   },
   COTTAGE: {
     id: "COTTAGE",
     title: "Vikendica",
     hasGardenSize: true,
-    priceSliderOptions: PRICE_SLIDER_OPTIONS,
+    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   }
@@ -133,14 +160,31 @@ const AD_STATUS = {
 };
 
 const AD_AGENCY = {
-  OLX: "OLX"
+  OLX: "OLX",
+  RENTAL: "RENTAL",
+  PROSTOR: "PROSTOR",
+  AKTIDO: "AKTIDO"
 };
 
 const CRAWLER_AD_TYPE = {
   NONE: 0,
   ALL: 1,
   ONLY_SELL: 2,
-  ONLY_RENT: 3
+  ONLY_RENT: 3,
+  ONLY_REQUEST: 4
+};
+
+const EMAIL_FREQUENCY = {
+  ASAP: {
+    id: 1,
+    stringId: "ASAP",
+    title: "Odmah"
+  },
+  DAILY: {
+    id: 2,
+    stringId: "DAILY",
+    title: "Jednom dnevno"
+  }
 };
 
 module.exports = {
@@ -148,5 +192,6 @@ module.exports = {
   AD_CATEGORY,
   AD_STATUS,
   AD_AGENCY,
-  CRAWLER_AD_TYPE
+  CRAWLER_AD_TYPE,
+  EMAIL_FREQUENCY
 };

app/config/appConfig.js

@@ -28,6 +28,8 @@ const MAX_REAL_ESTATES_IN_EMAIL =
 const MAX_REAL_ESTATES_IN_FIRST_EMAIL =
   parseInt(process.env.MAX_REAL_ESTATES_IN_FIRST_EMAIL) || 5;
 
+const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -36,5 +38,6 @@ module.exports = {
   STOP_CRAWLER,
   AWS_EMAIL_CONFIG,
   MAX_REAL_ESTATES_IN_EMAIL,
-  MAX_REAL_ESTATES_IN_FIRST_EMAIL
+  MAX_REAL_ESTATES_IN_FIRST_EMAIL,
+  PRINT_CRAWLER_DEBUG
 };

app/controllers/queryReview.js

@@ -3,11 +3,12 @@ const { isValidEmail } = require("../helpers/email");
 const {
   notifyForNewSearchRequest
 } = require("../services/notificationService");
-const { AD_CATEGORY } = require("../common/enums");
+const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-const getQueryReviewData = searchRequest => {
+const getQueryReviewTableData = searchRequest => {
   const {
     id,
+    adType,
     realEstateType,
     sizeMin,
     sizeMax,
@@ -22,8 +23,21 @@ const getQueryReviewData = searchRequest => {
     ? realEstateTypeObject.hasGardenSize
     : false;
 
+  let adTypeTitle = "";
+  switch (adType) {
+    case AD_TYPE.AD_TYPE_SALE.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
+      break;
+    case AD_TYPE.AD_TYPE_RENT.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
+      break;
+    default:
+      adTypeTitle = "-";
+      break;
+  }
+
   const realEstateTypeTitle = realEstateTypeObject
-    ? realEstateTypeObject.title
+    ? `[${adTypeTitle}] ${realEstateTypeObject.title}`
     : "-";
 
   const locationTitle = "Promjenite lokaciju";
@@ -73,15 +87,26 @@ const getQueryReview = async (req, res) => {
   const title = "Da li je ovo to što ste tražili ?";
   const nextStep = req.query.nextStep;
   const error = req.query.error;
-  const queryReviewData = getQueryReviewData(searchRequest);
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
   const email = searchRequest.email;
+  let selectedEmailFrequency;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.ASAP.id;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      selectedEmailFrequency = EMAIL_FREQUENCY.DAILY.id;
+      break;
+  }
 
   res.render("queryReview", {
     nextStep,
-    queryReviewData,
+    queryReviewTableData,
     title,
     email,
-    error
+    selectedEmailFrequency,
+    error,
+    EMAIL_FREQUENCY
   });
 };
 
@@ -93,17 +118,26 @@ const postQueryReview = async (req, res) => {
   }
 
   const nextStep = req.query.nextStep || "/ponovo";
+  const emailFrequency =
+    parseInt(req.body.emailFrequency) || EMAIL_FREQUENCY.ASAP.id;
   const emailInput = req.body.email;
   const emailConfirmInput = req.body.confirmEmail;
   const title = "Da li je ovo to što ste tražili ?";
-  const queryReviewData = getQueryReviewData(searchRequest);
+  const queryReviewTableData = getQueryReviewTableData(searchRequest);
+
+  let emailFrequencyStringId = EMAIL_FREQUENCY.ASAP.stringId;
+  if (emailFrequency === EMAIL_FREQUENCY.DAILY.id) {
+    emailFrequencyStringId = EMAIL_FREQUENCY.DAILY.stringId;
+  }
+
+  searchRequest.emailFrequency = emailFrequencyStringId;
 
   if (emailInput !== emailConfirmInput) {
     const error = "Greška ! Unešeni emailovi nisu isti";
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;
@@ -114,7 +148,7 @@ const postQueryReview = async (req, res) => {
     res.render("queryReview", {
       error,
       title,
-      queryReviewData,
+      queryReviewTableData,
       email: ""
     });
     return;
@@ -122,9 +156,39 @@ const postQueryReview = async (req, res) => {
 
   searchRequest.email = emailInput;
   searchRequest.subscribed = true;
-  await searchRequest.save();
 
-  await notifyForNewSearchRequest(searchRequest);
+  try {
+    await searchRequest.save();
+  } catch (e) {
+    console.log("[ERROR] Failed to save search request !", e);
+    console.log("Search request : ", searchRequest);
+    const error =
+      "Greška ! Nismo uspjeli kreirati zahtjev za Vašu pretragu. Molimo pokuštajte ponovo";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
+
+  try {
+    await notifyForNewSearchRequest(searchRequest);
+  } catch (e) {
+    console.log("[ERROR] Failed to send initial welcome email", e);
+    console.log("Search request : ", searchRequest);
+
+    const error =
+      "Greška ! Nismo uspjeli poslati email na Vašu adresu, pokušajte sa drugom email adresom";
+    res.render("queryReview", {
+      error,
+      title,
+      queryReviewTableData,
+      email: ""
+    });
+    return;
+  }
 
   res.redirect(nextStep);
 };

app/controllers/realEstateFilters.js

@@ -1,5 +1,5 @@
 const { currentSearchRequest } = require("../helpers/url");
-const { AD_CATEGORY } = require("../common/enums");
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
 
 const getFilters = async (req, res) => {
   const searchRequest = await currentSearchRequest(req);
@@ -12,6 +12,7 @@ const getFilters = async (req, res) => {
   const title = "Filteri za pretraživanje";
 
   const {
+    adType,
     realEstateType,
     priceMin,
     priceMax,
@@ -24,11 +25,22 @@ const getFilters = async (req, res) => {
 
   const {
     hasGardenSize,
-    priceSliderOptions,
+    priceSliderOptionsSale,
+    priceSliderOptionsRent,
     sizeSliderOptions,
     gardenSizeSliderOptions
   } = category;
 
+  let priceSliderOptions;
+  if (adType === AD_TYPE.AD_TYPE_SALE.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsSale);
+  } else if (adType === AD_TYPE.AD_TYPE_RENT.stringId) {
+    priceSliderOptions = Object.assign({}, priceSliderOptionsRent);
+  } else {
+    res.render("notFound", { title: " " });
+    return;
+  }
+
   if (priceMin || priceMax) {
     priceSliderOptions.start = [priceMin, priceMax];
   }
@@ -61,10 +73,10 @@ const postFilters = async (req, res) => {
   const nextStepPage = req.query.nextStep || "pregled";
   const nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
 
-  const priceMin = parseInt(req.body.priceFilterMin) || 0;
-  const priceMax = parseInt(req.body.priceFilterMax) || 0;
-  const sizeMin = parseInt(req.body.sizeFilterMin) || 0;
-  const sizeMax = parseInt(req.body.sizeFilterMax) || 0;
+  const priceMin = parseInt(req.body.priceMin) || 0;
+  const priceMax = parseInt(req.body.priceMax) || 0;
+  const sizeMin = parseInt(req.body.sizeMin) || 0;
+  const sizeMax = parseInt(req.body.sizeMax) || 0;
 
   //TODO: Filter validation
 
@@ -74,11 +86,11 @@ const postFilters = async (req, res) => {
   searchRequest.sizeMax = sizeMax;
 
   if (
-    req.body.gardenSizeFilterMin !== undefined &&
-    req.body.gardenSizeFilterMax !== undefined
+    req.body.gardenSizeMin !== undefined &&
+    req.body.gardenSizeMax !== undefined
   ) {
-    const gardenSizeMin = parseInt(req.body.gardenSizeFilterMin);
-    const gardenSizeMax = parseInt(req.body.gardenSizeFilterMax);
+    const gardenSizeMin = parseInt(req.body.gardenSizeMin);
+    const gardenSizeMax = parseInt(req.body.gardenSizeMax);
 
     //TODO: Filter validation
 

app/controllers/realEstateTypes.js

@@ -1,33 +1,68 @@
 const { currentSearchRequest } = require("../helpers/url");
 const { createSearchRequest } = require("../helpers/db/searchRequest");
 
-const { AD_CATEGORY } = require("../common/enums");
+const { AD_CATEGORY, AD_TYPE } = require("../common/enums");
+
+const getRealEstateTypes = async (req, res) => {
+  const searchRequest = await currentSearchRequest(req);
 
-const getRealEstateTypes = (req, res) => {
   const title = "Koju nekretninu tražite?";
-  const realEstateTypes = Object.keys(AD_CATEGORY).map(
-    category => AD_CATEGORY[category]
-  );
-  res.render("realEstateType", { realEstateTypes, title });
+  let selectedAdType = AD_TYPE.AD_TYPE_SALE.id;
+  if (
+    searchRequest &&
+    searchRequest.adType &&
+    searchRequest.adType === AD_TYPE.AD_TYPE_RENT.stringId
+  ) {
+    selectedAdType = AD_TYPE.AD_TYPE_RENT.id;
+  }
+  const realEstateTypes = Object.keys(AD_CATEGORY)
+    .map(category => AD_CATEGORY[category])
+    .filter(category => category.title);
+
+  res.render("realEstateType", {
+    selectedAdType,
+    realEstateTypes,
+    title,
+    AD_TYPE
+  });
 };
 
 const postRealEstateTypes = async (req, res) => {
   const searchRequest = await currentSearchRequest(req);
 
-  //TODO: check if selected real estate type is valid
+  const adType = parseInt(req.body.adType);
+
+  const adTypeStringIds = {
+    [AD_TYPE.AD_TYPE_SALE.id]: AD_TYPE.AD_TYPE_SALE.stringId,
+    [AD_TYPE.AD_TYPE_RENT.id]: AD_TYPE.AD_TYPE_RENT.stringId
+  };
+
+  const adTypeStringId =
+    adTypeStringIds[adType] || AD_TYPE.AD_TYPE_SALE.stringId;
+
+  const validRealEstateTypes = Object.keys(AD_CATEGORY).filter(
+    category => !!AD_CATEGORY[category].title
+  );
+
   const selectedRealEstateType = req.body.realEstateType || null;
+  if (validRealEstateTypes.indexOf(selectedRealEstateType) === -1) {
+    res.render("notFound", { title: " " });
+    return;
+  }
 
   const nextStepPage = req.query.nextStep || "lokacija";
 
   let nextStepUrl = "";
   if (searchRequest && searchRequest.id) {
     nextStepUrl = `/${nextStepPage}/${searchRequest.id}`;
+    searchRequest.adType = adTypeStringId;
     searchRequest.realEstateType = selectedRealEstateType;
 
     await searchRequest.save();
   } else {
     try {
       const newSearchRequest = await createSearchRequest({
+        adType: adTypeStringId,
         realEstateType: selectedRealEstateType
       });
 

app/controllers/unsubscribe.js

@@ -10,6 +10,8 @@ const getUnsubscribe = async (req, res) => {
   }
 
   searchRequest.subscribed = false;
+  searchRequest.deletedEmail = searchRequest.email;
+  searchRequest.email = "";
   await searchRequest.save();
 
   res.render("unsubscribe", { nextStep: "/vrstanekretnine", title });

app/crawler/crawl.js

@@ -5,31 +5,75 @@
    All environment specific configuration is read here and
    passed to the crawlers and savers.
 */
-const OlxCrawler = require("./specific/olx");
-const { OLX_CONFIG } = require("./crawlerConfig");
+const OlxCrawler = require("./specificCrawlers/olx");
+const RentalCrawler = require("./specificCrawlers/rental");
+const ProstorCrawler = require("./specificCrawlers/prostor");
+const AktidoCrawler = require("./specificCrawlers/aktido");
+
+const {
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG
+} = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
-const crawlers = [
-  new OlxCrawler(
-    [new PostgresSaver()],
-    OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
-    OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
-    OLX_CONFIG.OLX_MAX_PAGES,
-    OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
-    OLX_CONFIG.OLX_IGNORED_USERNAMES,
-    OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
-  )
-];
-
 async function crawlAll() {
-  for (let crawler of crawlers) {
+  const postgresSaver = new PostgresSaver();
+
+  const crawlers = [
+    new OlxCrawler(
+      [postgresSaver],
+      OLX_CONFIG.OLX_CRAWLER_AD_TYPE,
+      OLX_CONFIG.OLX_CRAWLER_AD_CATEGORIES,
+      OLX_CONFIG.OLX_MAX_PAGES,
+      OLX_CONFIG.OLX_MAX_RESULTS_PER_PAGE,
+      OLX_CONFIG.OLX_IGNORED_USERNAMES,
+      OLX_CONFIG.OLX_DELAY_BETWEEN_PAGES
+    ),
+    new RentalCrawler(
+      [postgresSaver],
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_TYPE,
+      RENTAL_CONFIG.RENTAL_CRAWLER_AD_CATEGORIES,
+      RENTAL_CONFIG.RENTAL_MAX_PAGES,
+      RENTAL_CONFIG.RENTAL_MAX_RESULTS_PER_PAGE,
+      RENTAL_CONFIG.RENTAL_IGNORED_USERNAMES,
+      RENTAL_CONFIG.RENTAL_DELAY_BETWEEN_PAGES
+    ),
+    new ProstorCrawler(
+      [postgresSaver],
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_TYPE,
+      PROSTOR_CONFIG.PROSTOR_CRAWLER_AD_CATEGORIES,
+      PROSTOR_CONFIG.PROSTOR_MAX_PAGES,
+      PROSTOR_CONFIG.PROSTOR_MAX_RESULTS_PER_PAGE,
+      PROSTOR_CONFIG.PROSTOR_IGNORED_USERNAMES,
+      PROSTOR_CONFIG.PROSTOR_DELAY_BETWEEN_PAGES
+    ),
+    new AktidoCrawler(
+      [postgresSaver],
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_TYPE,
+      AKTIDO_CONFIG.AKTIDO_CRAWLER_AD_CATEGORIES,
+      AKTIDO_CONFIG.AKTIDO_MAX_PAGES,
+      AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
+      AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
+      AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
+    )
+  ];
+
+  const newRealEstates = [];
+
+  for (const crawler of crawlers) {
     try {
-      return await crawler.crawl();
+      const newRealEstatesFromSingleCrawler = await crawler.crawl();
+      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
+        newRealEstates.push(...newRealEstatesFromSingleCrawler);
+      }
     } catch (e) {
       console.log("Error crawling. Trying next crawler! ", e);
-      return [];
     }
   }
+
+  return newRealEstates;
 }
 
 module.exports = {

app/crawler/crawlerConfig.js

@@ -1,42 +1,14 @@
 "use strict";
 require("dotenv").config({ path: __dirname + "/./../../.env" });
-const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../common/enums");
 
-const olxCrawlerAdType =
-  process.env.OLX_CRAWLER_AD_TYPE !== undefined
-    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
-    : null;
-
-const olxParsedCrawlerAdCategories =
-  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
-    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
-        category.trim()
-      )
-    : ["FLAT", "HOUSE"];
-
-const olxIgnoredUsernames =
-  process.env.OLX_IGNORED_USERNAMES !== undefined
-    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
-        username.trim()
-      )
-    : [];
-
-const transformedCrawlerAdCategories = olxParsedCrawlerAdCategories
-  .map(categoryName =>
-    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
-  )
-  .filter(category => !!category);
-
-const OLX_CONFIG = {
-  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
-  OLX_MAX_RESULTS_PER_PAGE:
-    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
-  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  OLX_CRAWLER_AD_CATEGORIES: transformedCrawlerAdCategories,
-  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
-  OLX_DELAY_BETWEEN_PAGES: parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000
-};
+const OLX_CONFIG = require("./specificConfigs/olx");
+const RENTAL_CONFIG = require("./specificConfigs/rental");
+const PROSTOR_CONFIG = require("./specificConfigs/prostor");
+const AKTIDO_CONFIG = require("./specificConfigs/aktido");
 
 module.exports = {
-  OLX_CONFIG
+  OLX_CONFIG,
+  RENTAL_CONFIG,
+  PROSTOR_CONFIG,
+  AKTIDO_CONFIG
 };

app/crawler/specificConfigs/aktido.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const aktidoCrawlerAdType =
+  process.env.AKTIDO_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.AKTIDO_CRAWLER_AD_TYPE]
+    : null;
+
+const aktidoParsedCrawlerAdCategories =
+  process.env.AKTIDO_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.AKTIDO_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const aktidoIgnoredUsernames = [];
+
+const transformedAktidoCrawlerAdCategories = aktidoParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  AKTIDO_MAX_PAGES: parseInt(process.env.AKTIDO_MAX_PAGES) || 500,
+  AKTIDO_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.AKTIDO_MAX_RESULTS_PER_PAGE) || 50,
+  AKTIDO_CRAWLER_AD_TYPE: aktidoCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  AKTIDO_CRAWLER_AD_CATEGORIES: transformedAktidoCrawlerAdCategories,
+  AKTIDO_IGNORED_USERNAMES: aktidoIgnoredUsernames || [],
+  AKTIDO_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.AKTIDO_DELAY_BETWEEN_PAGES) || 1000,
+  AKTIDO_FORCE_CRAWL: !!parseInt(process.env.AKTIDO_FORCE_CRAWL)
+};

app/crawler/specificConfigs/olx.js

@@ -0,0 +1,39 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const olxCrawlerAdType =
+  process.env.OLX_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.OLX_CRAWLER_AD_TYPE]
+    : null;
+
+const olxParsedCrawlerAdCategories =
+  process.env.OLX_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.OLX_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const olxIgnoredUsernames =
+  process.env.OLX_IGNORED_USERNAMES !== undefined
+    ? process.env.OLX_IGNORED_USERNAMES.split(",").map(username =>
+        username.trim()
+      )
+    : [];
+
+const transformedOlxCrawlerAdCategories = olxParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  OLX_MAX_PAGES: parseInt(process.env.OLX_MAX_PAGES) || 500,
+  OLX_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.OLX_MAX_RESULTS_PER_PAGE) || 50,
+  OLX_CRAWLER_AD_TYPE: olxCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  OLX_CRAWLER_AD_CATEGORIES: transformedOlxCrawlerAdCategories,
+  OLX_IGNORED_USERNAMES: olxIgnoredUsernames || [],
+  OLX_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.OLX_DELAY_BETWEEN_PAGES) || 1000,
+  OLX_FORCE_CRAWL: !!parseInt(process.env.OLX_FORCE_CRAWL)
+};

app/crawler/specificConfigs/prostor.js

@@ -0,0 +1,33 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const prostorCrawlerAdType =
+  process.env.PROSTOR_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.PROSTOR_CRAWLER_AD_TYPE]
+    : null;
+
+const prostorParsedCrawlerAdCategories =
+  process.env.PROSTOR_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.PROSTOR_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const prostorIgnoredUsernames = [];
+
+const transformedProstorCrawlerAdCategories = prostorParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  PROSTOR_MAX_PAGES: parseInt(process.env.PROSTOR_MAX_PAGES) || 100,
+  PROSTOR_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.PROSTOR_MAX_RESULTS_PER_PAGE) || 5000,
+  PROSTOR_CRAWLER_AD_TYPE: prostorCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  PROSTOR_CRAWLER_AD_CATEGORIES: transformedProstorCrawlerAdCategories,
+  PROSTOR_IGNORED_USERNAMES: prostorIgnoredUsernames || [],
+  PROSTOR_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.PROSTOR_DELAY_BETWEEN_PAGES) || 1000
+};

app/crawler/specificConfigs/rental.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const rentalCrawlerAdType =
+  process.env.RENTAL_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.RENTAL_CRAWLER_AD_TYPE]
+    : null;
+
+const rentalParsedCrawlerAdCategories =
+  process.env.RENTAL_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.RENTAL_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const rentalIgnoredUsernames = [];
+
+const transformedRentalCrawlerAdCategories = rentalParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  RENTAL_MAX_PAGES: parseInt(process.env.RENTAL_MAX_PAGES) || 500,
+  RENTAL_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.RENTAL_MAX_RESULTS_PER_PAGE) || 50,
+  RENTAL_CRAWLER_AD_TYPE: rentalCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  RENTAL_CRAWLER_AD_CATEGORIES: transformedRentalCrawlerAdCategories,
+  RENTAL_IGNORED_USERNAMES: rentalIgnoredUsernames || [],
+  RENTAL_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.RENTAL_DELAY_BETWEEN_PAGES) || 1000,
+  RENTAL_FORCE_CRAWL: !!parseInt(process.env.RENTAL_FORCE_CRAWL)
+};

app/crawler/specificCrawlers/aktido.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const AKTIDO_ENUMS = {
+  AKTIDO_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  AKTIDO_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  AKTIDO_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  AKTIDO_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { AKTIDO_FORCE_CRAWL } = require("../specificConfigs/aktido");
+
+class AktidoCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.aktido.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !AKTIDO_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = AKTIDO_ENUMS.AKTIDO_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = AKTIDO_ENUMS.AKTIDO_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[AKTIDO] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".moreInfo")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[AKTIDO] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        const aktidoId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromAktidoId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromAktidoActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          AKTIDO_ENUMS.AKTIDO_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          AKTIDO_ENUMS.AKTIDO_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const adStatus = AD_STATUS.STATUS_NORMAL;
+
+        const data = {
+          url,
+          agencyObjectId: aktidoId,
+          originAgencyName: AD_AGENCY.AKTIDO,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString()
+        };
+
+        return data;
+      } else {
+        console.log("[AKTIDO] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[AKTIDO] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromAktidoId(aktidoCategoryId) {
+    switch (aktidoCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromAktidoActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = AktidoCrawler;

app/crawler/specificCrawlers/olx.js

@@ -13,13 +13,17 @@ const {
   CRAWLER_AD_TYPE
 } = require("../../common/enums");
 
-const { DEFAULT_TIMEZONE } = require("../../config/appConfig");
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
 
 const OLX_ENUMS = {
   OLX_AD_TYPE: {
     [CRAWLER_AD_TYPE.ALL]: "",
     [CRAWLER_AD_TYPE.ONLY_SELL]: "&vrsta=samoprodaja",
-    [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje"
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&vrsta=samoizdavanje",
+    [CRAWLER_AD_TYPE.ONLY_REQUEST]: "&vrsta=samopotraznja"
   },
   OLX_AD_CATEGORY: {
     [AD_CATEGORY.FLAT.id]: "&kategorija=23",
@@ -35,6 +39,8 @@ const OLX_ENUMS = {
   OLX_RENEWED_DATE_FORMAT: "DD.MM.YYYY. u HH:mm"
 };
 
+const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
+
 class OlxCrawler {
   constructor(
     savers = [],
@@ -96,7 +102,7 @@ class OlxCrawler {
                 "minute"
               );
 
-              if (stopCrawlingThisCategory) {
+              if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
                 generatorsToRemove[index] = true;
                 // console.log("\tGenerator ", index + 1, "has no more new ads");
                 break;
@@ -131,7 +137,7 @@ class OlxCrawler {
 
     const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
     const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
-    if (urlAdTypePart && urlCategoryPart) {
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
       while (true) {
         const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
         const singlePageResults = await this.indexSinglePage(
@@ -156,6 +162,10 @@ class OlxCrawler {
   }
 
   async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[OLX] Index page : ", url);
+    }
+
     try {
       const res = await fetch(url);
       const body = await res.text();
@@ -205,7 +215,7 @@ class OlxCrawler {
         title: "#naslovartikla",
         descriptions: ".artikal_detaljniopis_tekst",
         category:
-          "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(3) > div > span:nth-child(3) > a > span"
+          "#artikal_glavni_div > div.artikal_lijevo > div.artikal_kat > div > span:nth-child(3) > a > span"
       };
 
       const username = $(propertySelectors.username)
@@ -377,7 +387,7 @@ class OlxCrawler {
       //=========================================
       const parsedCategory = this.getAdCategoryId(category);
       if (!parsedCategory) {
-        throw { message: "Unknown ad category" };
+        throw { message: `Unknown ad category [${category}]` };
       }
 
       const parsedAdType = this.getAdTypeId(adType);
@@ -465,9 +475,11 @@ class OlxCrawler {
   getAdTypeId(adTypeText) {
     switch (adTypeText) {
       case "Prodaja":
-        return AD_TYPE.AD_TYPE_SALE;
+        return AD_TYPE.AD_TYPE_SALE.stringId;
       case "Izdavanje":
-        return AD_TYPE.AD_TYPE_RENT;
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      case "Potražnja":
+        return AD_TYPE.AD_TYPE_REQUEST.stringId;
       default:
         return undefined;
     }

app/crawler/specificCrawlers/prostor.js

@@ -0,0 +1,252 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const { PRINT_CRAWLER_DEBUG } = require("../../config/appConfig");
+
+const PROSTOR_ENUMS = {
+  PROSTOR_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&action=0",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&action=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&action=2"
+  },
+  PROSTOR_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "&type=7",
+    [AD_CATEGORY.HOUSE.id]: "&type=8",
+    [AD_CATEGORY.LAND.id]: "&type=10",
+    [AD_CATEGORY.OFFICE.id]: "&type=9",
+    [AD_CATEGORY.APARTMENT.id]: "&type=11",
+    [AD_CATEGORY.GARAGE.id]: "&type=14"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  PROSTOR_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  PROSTOR_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+class ProstorCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://prostor.ba/pretraga";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      for (const adCategory of crawlAdCategories) {
+        const urlAdTypePart =
+          PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
+        const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
+        if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+          const urlPageToCrawl = `${this.baseUrl}?remove_sold=1${urlAdTypePart}${urlCategoryPart}`;
+          const singleCategoryResults = await this.extractRealEstates(
+            urlPageToCrawl
+          );
+
+          const resultsSubset = singleCategoryResults.slice(
+            0,
+            this.maxResultsPerPage
+          );
+
+          const saveResults = await this.saveCrawledResults(resultsSubset);
+          const { newRecords } = saveResults;
+          newRealEstates.push(...newRecords);
+        }
+      }
+    }
+    return newRealEstates;
+  }
+
+  async extractRealEstates(url) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[PROSTOR] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+
+      const scriptElement = $(
+        "body > div > div.container-fluid > script:nth-child(7)"
+      );
+
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        const scriptData = scriptElement[0].children[0].data;
+
+        try {
+          // script element data contains JS code and we need to extract only data for realEstates
+          // data string starts with : var map; var markers = [{"r ...
+          // so we remove first 23 characters
+          //
+          // real estate JSON data ends with      ...}, ]; map = new...
+          // so we need to find index of that substring to know where to stop
+          // we will NOT include trailing comma because it breaks JSON parse, so we have to close ] bracket manually
+
+          const jsonEndIndex = scriptData.indexOf(", ]; map = new");
+          if (jsonEndIndex > -1) {
+            const jsonData = scriptData.substring(23, jsonEndIndex) + "]";
+            const realEstates = JSON.parse(jsonData);
+
+            const transformedRealEstates = [];
+
+            for (const realEstate of realEstates) {
+              const transformedRealEstate = ProstorCrawler.transformRealEstateData(
+                realEstate
+              );
+              if (transformedRealEstate) {
+                transformedRealEstates.push(transformedRealEstate);
+              }
+            }
+
+            return transformedRealEstates;
+          } else {
+            throw {
+              message: "Something is wrong with JSON data or data is moved"
+            };
+          }
+        } catch (e) {
+          console.log(e);
+          throw { message: "Can't find ad data JSON" };
+        }
+      }
+    } catch (e) {
+      console.error("[PROSTOR] Exception caught:", e.message);
+      return [];
+    }
+  }
+
+  static transformRealEstateData(realEstateData) {
+    try {
+      const { lat, lng, property_name, price, size, link } = realEstateData;
+
+      // link contains part of the URL in the format of : /prodaja/stan/stup/9556
+      // general form is : /actionType/realEstateType/location/realEstateID
+      // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
+
+      const linkParts = link.split("/");
+
+      const adType = ProstorCrawler.getAdTypeId(linkParts[1]);
+      const realEstateType = ProstorCrawler.getAdCategoryId(linkParts[2]);
+      const prostorId = linkParts[4];
+      const url = `https://prostor.ba${link}`;
+
+      if (!adType || !realEstateType || !prostorId) {
+        return null;
+      }
+
+      const adStatus = AD_STATUS.STATUS_NORMAL;
+      const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
+      const parsedArea = parseFloat(size);
+
+      const data = {
+        url,
+        agencyObjectId: prostorId,
+        originAgencyName: AD_AGENCY.PROSTOR,
+        realEstateType,
+        adType,
+        title: property_name,
+        price: parsedPrice,
+        area: parsedArea,
+        gardenSize: null,
+        shortDescription: "",
+        longDescription: "",
+        streetNumber: 0,
+        streetName: "",
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat: lat,
+        locationLong: lng,
+        adStatus,
+        publishedDate: null,
+        renewedDate: null
+      };
+
+      return data;
+    } catch (e) {
+      console.error(
+        "[PROSTOR] Exception caught: " + e.message,
+        "\r\nURL:",
+        url
+      );
+      return null;
+    }
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  static getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "stan":
+        return AD_CATEGORY.FLAT.id;
+      case "kuca":
+        return AD_CATEGORY.HOUSE.id;
+      case "apartman":
+        return AD_CATEGORY.APARTMENT.id;
+      case "poslovni-prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "garaza":
+        return AD_CATEGORY.GARAGE.id;
+      case "zemljiste":
+        return AD_CATEGORY.LAND.id;
+      default:
+        return undefined;
+    }
+  }
+
+  static getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "prodaja":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "najam":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = ProstorCrawler;

app/crawler/specificCrawlers/rental.js

@@ -0,0 +1,370 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const Promise = require("bluebird");
+const moment = require("moment-timezone");
+const htmlToText = require("html-to-text");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE
+} = require("../../common/enums");
+
+const {
+  DEFAULT_TIMEZONE,
+  PRINT_CRAWLER_DEBUG
+} = require("../../config/appConfig");
+
+const RENTAL_ENUMS = {
+  RENTAL_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "/prodaja-1/najam-2",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "/prodaja-1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "/najam-2"
+  },
+  RENTAL_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "",
+    [AD_CATEGORY.FLAT.id]: "/tip-2",
+    [AD_CATEGORY.HOUSE.id]: "/tip-1",
+    [AD_CATEGORY.LAND.id]: "/tip-5",
+    [AD_CATEGORY.OFFICE.id]: "/tip-4",
+    [AD_CATEGORY.APARTMENT.id]: "/tip-3",
+    [AD_CATEGORY.GARAGE.id]: "/tip-6"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  },
+  RENTAL_PUBLISHED_DATE_FORMAT: "YYYY-MM-DD HH:mm:ss",
+  RENTAL_RENEWED_DATE_FORMAT: "YYYY-MM-DD u HH:mm:ss"
+};
+
+const { RENTAL_FORCE_CRAWL } = require("../specificConfigs/rental");
+
+class RentalCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 1000,
+    maxResultsPerPage = 100,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.rental.ba/pretraga/sortiraj-date_DESC";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxPages = maxPages;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !RENTAL_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = RENTAL_ENUMS.RENTAL_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = RENTAL_ENUMS.RENTAL_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}/stranica-${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[RENTAL] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $(
+        "body > div > div.container > div.row > div.col-xs-12.col-sm-12.col-md-12.col-lg-9.content-main > div.row.box-items.group-grid-view"
+      )
+        .find(".pull-right")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let actualNoOfResults =
+        hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[RENTAL] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url) {
+    // console.log("[RENTAL] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      const mapElementParent = $(".box-map").parent();
+      const scriptElement = $("script", mapElementParent);
+      if (
+        scriptElement[0] &&
+        scriptElement[0].children &&
+        scriptElement[0].children[0] &&
+        scriptElement[0].children[0].data
+      ) {
+        let extractedData;
+        try {
+          //data string starts with : var json_map_data = [{"r ...
+          //so we remove first 20 characters
+
+          const jsonData = scriptElement[0].children[0].data.substring(20);
+          const parsedJsonData = JSON.parse(jsonData);
+          extractedData = parsedJsonData[0];
+        } catch (e) {
+          throw { message: "Can't find ad data JSON" };
+        }
+
+        const rentalId = extractedData["re_realEstates_id"];
+        const adCategory = this.getKiviCategoryIdFromRentalId(
+          parseInt(extractedData["re_types_id"])
+        );
+        if (!adCategory) {
+          throw {
+            message: `Invalid category : ${extractedData["re_types_id"]}`
+          };
+        }
+        const adType = this.getKiviAdTypeFromRentalActionId(
+          parseInt(extractedData["re_action_id"])
+        );
+        if (!adType) {
+          throw {
+            message: `Invalid ad type : ${extractedData["re_action_id"]}`
+          };
+        }
+
+        const title = extractedData["re_realEstates_portalName"];
+        const extractedPrice = parseFloat(
+          extractedData["re_realEstates_price"]
+        );
+        const price = extractedPrice ? extractedPrice : null;
+        const area = parseFloat(extractedData["re_realEstates_area"]);
+        const gardenSize = parseFloat(
+          extractedData["re_realEstates_fieldArea"]
+        );
+        const longDescription = htmlToText.fromString(
+          extractedData["re_realEstates_description"]
+        );
+        const locationLong = extractedData["re_realEstates_longitude"];
+        const locationLat = extractedData["re_realEstates_latitude"];
+        const publishedDateMoment = moment.tz(
+          extractedData["re_realEstates_inserted"],
+          RENTAL_ENUMS.RENTAL_PUBLISHED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!publishedDateMoment.isValid()) {
+          throw {
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
+          };
+        }
+
+        const renewedDateMoment = moment.tz(
+          extractedData["re_realEstates_edited"],
+          RENTAL_ENUMS.RENTAL_RENEWED_DATE_FORMAT,
+          DEFAULT_TIMEZONE
+        );
+        if (!renewedDateMoment.isValid()) {
+          throw {
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
+          };
+        }
+
+        const adStatus = AD_STATUS.STATUS_NORMAL;
+
+        const data = {
+          url,
+          agencyObjectId: rentalId,
+          originAgencyName: AD_AGENCY.RENTAL,
+          realEstateType: adCategory,
+          adType,
+          title,
+          price,
+          area,
+          gardenSize,
+          shortDescription: "",
+          longDescription: longDescription,
+          streetNumber: 0,
+          streetName: "",
+          locality: "",
+          municipality: "",
+          city: "",
+          region: "",
+          entity: "",
+          country: "",
+          locationLat,
+          locationLong,
+          adStatus,
+          publishedDate: publishedDateMoment.toISOString(),
+          renewedDate: renewedDateMoment.toISOString()
+        };
+
+        return data;
+      } else {
+        console.log("[RENTAL] No JSON data for this ad : ", url);
+        return null;
+      }
+    } catch (e) {
+      console.error("[RENTAL] Exception caught: " + e.message, "\r\nURL:", url);
+      return null;
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getKiviCategoryIdFromRentalId(rentalCategoryId) {
+    switch (rentalCategoryId) {
+      case 1:
+        return AD_CATEGORY.HOUSE.id;
+      case 2:
+        return AD_CATEGORY.FLAT.id;
+      case 3:
+        return AD_CATEGORY.APARTMENT.id;
+      case 4:
+        return AD_CATEGORY.OFFICE.id;
+      case 5:
+        return AD_CATEGORY.LAND.id;
+      case 6:
+        return AD_CATEGORY.GARAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getKiviAdTypeFromRentalActionId(actionId) {
+    switch (actionId) {
+      case 1:
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case 2:
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return await savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = RentalCrawler;

app/helpers/db/searchRequestMatch.js

@@ -24,6 +24,23 @@ const findRealEstatesForSearchRequest = async searchRequestId => {
   return matchingRealEstates;
 };
 
+const findNotNotifiedMatches = async () => {
+  const query = {
+    notified: false
+  };
+
+  const searchRequestsModel = { model: db.SearchRequest, as: "searchRequests" };
+  const realEstateModel = { model: db.RealEstate, as: "realEstates" };
+  const include = [searchRequestsModel, realEstateModel];
+
+  const matchingRecords = await db.SearchRequestMatch.findAll({
+    where: query,
+    include
+  });
+
+  return matchingRecords;
+};
+
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
     ignoreDuplicates: true
@@ -32,5 +49,6 @@ const addMatches = async matchingRecords => {
 
 module.exports = {
   findRealEstatesForSearchRequest,
-  addMatches
+  addMatches,
+  findNotNotifiedMatches
 };

app/helpers/email.js

@@ -1,5 +1,5 @@
 const isValidEmail = email => {
-  const simpleEmailRegex = /^.+@.+\..+$/;
+  const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
   return email && email.length < 250 && simpleEmailRegex.test(email);
 };
 

app/helpers/emailContentGenerator.js

@@ -20,7 +20,11 @@ const generateRealEstateLinks = realEstates => {
   return realEstateLinks;
 };
 
-const generateNotificationEmail = (realEstates, searchRequestId) => {
+const generateNotificationEmail = (
+  realEstates,
+  searchRequestId,
+  dailyNotification = false
+) => {
   const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
   const realEstatesToShow = truncateList
     ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
@@ -30,9 +34,20 @@ const generateNotificationEmail = (realEstates, searchRequestId) => {
   const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
   const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
   const emailFooter = generateEmailFooter(searchRequestId);
+  const asapMessageBody =
+    realEstates.length > 1
+      ? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
+      : "Pronašli smo nekretninu koja odgovara Vašoj pretrazi";
+
+  const dailyMessageBody =
+    realEstates.length > 1
+      ? "U posljednja 24h objavljene su sljedeće nekretnine koje odgovaraju uslovima Vaše pretrage"
+      : "U posljednja 24h objavljena je sljedeća nekretnina koja odgovara uslovima Vaše pretrage";
+
+  const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
 
   return `<h3>Zdravo</h3>
-  <h4>Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi</h4>
+  <h4>${messageBody}</h4>
   <div>
   ${realEstateLinks}
   <div/>

app/migrations/20191101113541-add-emailFrequency-column-to-searchRequest-table.js

@@ -0,0 +1,15 @@
+"use strict";
+const { EMAIL_FREQUENCY } = require("../common/enums");
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "emailFrequency", {
+      type: Sequelize.TEXT,
+      defaultValue: EMAIL_FREQUENCY.ASAP.stringId
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "emailFrequency");
+  }
+};

app/migrations/20191105111202-add-deletedEmail-column-to-searchRequest-table.js

@@ -0,0 +1,13 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "deletedEmail", {
+      type: Sequelize.TEXT
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "deletedEmail");
+  }
+};

app/models/searchRequest.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const { AD_TYPE } = require("../common/enums");
+const { AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
 module.exports = (sequelize, DataTypes) => {
   const SearchRequest = sequelize.define("SearchRequest", {
@@ -26,7 +26,7 @@ module.exports = (sequelize, DataTypes) => {
     adType: {
       type: DataTypes.TEXT,
       allowNull: false,
-      defaultValue: AD_TYPE.AD_TYPE_SALE
+      defaultValue: AD_TYPE.AD_TYPE_SALE.stringId
     },
     email: DataTypes.TEXT,
     locality: DataTypes.TEXT,
@@ -61,6 +61,14 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.BOOLEAN,
       defaultValue: false,
       allowNull: false
+    },
+    emailFrequency: {
+      type: DataTypes.TEXT,
+      defaultValue: EMAIL_FREQUENCY.ASAP.stringId,
+      allowNull: false
+    },
+    deletedEmail: {
+      type: DataTypes.TEXT
     }
   });
 

app/models/searchRequestMatch.js

@@ -44,6 +44,12 @@ module.exports = (sequelize, DataTypes) => {
   );
 
   SearchRequestMatch.associate = models => {
+    SearchRequestMatch.hasMany(models.SearchRequest, {
+      foreignKey: "id",
+      sourceKey: "searchRequestId",
+      targetKey: "id",
+      as: "searchRequests"
+    });
     SearchRequestMatch.hasMany(models.RealEstate, {
       foreignKey: "id",
       as: "realEstates"

app/npmScripts/npmDailyNotify.js

@@ -0,0 +1,8 @@
+"use strict";
+const {
+  notifyRequestsWithDailyOption
+} = require("../services/notificationService");
+
+(async () => {
+  await notifyRequestsWithDailyOption();
+})();

app/public/main.css

@@ -61,18 +61,6 @@ body {
   box-sizing: border-box;
 }
 
-#floating-panel {
-  top: 10px;
-  left: 25%;
-  z-index: 5;
-  background-color: #fff;
-  border: 1px solid #999;
-  text-align: center;
-  font-family: "Roboto", "sans-serif";
-  line-height: 30px;
-  padding: 5px 5px 5px 10px;
-}
-
 .btn:hover {
   background-color: white;
   color: #02adba;
@@ -95,7 +83,22 @@ h6.title {
   margin-right: 10px;
 }
 
+.full-width {
+  width: 100%;
+}
+
+strong {
+  font-weight: bold;
+}
+
 h3 {
   font-size: 15px;
   line-height: 1.5;
 }
+
+.sliderInputBox {
+  box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.4) !important;
+  border: 1px solid #02adba !important;
+  border-radius: 4px !important;
+  text-align: center;
+}

app/public/segment.css

@@ -0,0 +1,38 @@
+.segmented {
+  color: #02adba;
+  border: 1px solid #02adba;
+  border-radius: 4px;
+  display: inline-block;
+}
+.segmented label {
+  color: #02adba;
+}
+.segmented input:checked + .label {
+  background-color: #02adba;
+  color: white;
+}
+[type="radio"]:not(:checked) + span,
+[type="radio"]:checked + span {
+  padding-left: 30px;
+  padding-right: 30px;
+  height: 35px;
+  text-align: center;
+  display: inline-block;
+  line-height: 35px;
+  margin: 0px;
+  float: left;
+  cursor: pointer;
+  border-right: 1px solid #02adba;
+}
+
+.segmented :last-child .label {
+  border-right: none;
+}
+.segmented input {
+  display: none;
+}
+
+span.label:before,
+span.label:after {
+  display: none;
+}

app/services/notificationService.js

@@ -8,6 +8,7 @@ const {
   generateNewSearchRequestEmail,
   generateEmailSubject
 } = require("../helpers/emailContentGenerator");
+const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -29,34 +30,87 @@ const notifyForNewSearchRequest = async searchRequest => {
   await sendEmail(email, "Kivi - novi zahtjev za pretragu", emailContent);
 };
 
-const notifyMatches = async matches => {
+const notifyMatches = async (matches, dailyNotification = false) => {
   const searchRequestsToNotify = Object.keys(matches);
 
   const asyncSendEmailActions = [];
   for (const id of searchRequestsToNotify) {
-    const { searchRequest } = matches[id];
-    const { email } = searchRequest;
-    const allMatchingRealEstates = matches[id].realEstates || [];
-    if (allMatchingRealEstates.length > 0) {
-      const emailContent = generateNotificationEmail(
-        allMatchingRealEstates,
-        id
-      );
-      const emailSubject = generateEmailSubject(
-        allMatchingRealEstates.length,
-        allMatchingRealEstates[0].title
-      );
+    const { searchRequest, notifyNow } = matches[id];
+    const { email, subscribed } = searchRequest;
+    if (notifyNow && subscribed) {
+      const allMatchingRealEstates = matches[id].realEstates || [];
+      if (allMatchingRealEstates.length > 0) {
+        const emailContent = generateNotificationEmail(
+          allMatchingRealEstates,
+          id,
+          dailyNotification
+        );
+        const emailSubject = generateEmailSubject(
+          allMatchingRealEstates.length,
+          allMatchingRealEstates[0].title
+        );
 
-      const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
-      asyncSendEmailActions.push(sendEmailPromise);
-      sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+        const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
+        asyncSendEmailActions.push(sendEmailPromise);
+        sendEmailPromise.catch(err =>
+          console.log("[Email Sending Failed]", err)
+        );
+      }
     }
   }
 
   await Promise.all(asyncSendEmailActions);
 };
 
+const notifyRequestsWithDailyOption = async () => {
+  const notNotifiedSearchRequestMatches = await findNotNotifiedMatches();
+
+  const matches = {};
+
+  for (const searchRequestMatch of notNotifiedSearchRequestMatches) {
+    const { searchRequests, realEstates } = searchRequestMatch;
+
+    if (!Array.isArray(searchRequests) || searchRequests.length !== 1) {
+      // Something is wrong with this match
+      // (search request not found for specified search request id)
+      // OR
+      // there are multiple search requests with the same ID (this should never be the case !
+      // TODO: Maybe if association is defined better, this will be automatically only one object instead of array
+      continue;
+    }
+
+    if (!Array.isArray(realEstates) || realEstates.length !== 1) {
+      // Something is wrong with this match
+      // (real estate not found for specified real estate id)
+      // OR
+      // there are multiple real estates with the same ID (this should never be the case !
+      // TODO: Maybe if association is defined better, this will be automatically only one object instead of array
+      continue;
+    }
+
+    const searchRequest = searchRequests[0];
+    const realEstate = realEstates[0];
+    const searchRequestId = searchRequest.id;
+
+    if (!matches[searchRequestId]) {
+      matches[searchRequestId] = {
+        searchRequest,
+        realEstates: [],
+        notifyNow: true
+      };
+    }
+
+    matches[searchRequestId].realEstates.push(realEstate);
+
+    searchRequestMatch.notified = true;
+    searchRequestMatch.save();
+  }
+
+  await notifyMatches(matches, true);
+};
+
 module.exports = {
   notifyForNewRealEstates,
-  notifyForNewSearchRequest
+  notifyForNewSearchRequest,
+  notifyRequestsWithDailyOption
 };

app/services/searchMatchService.js

@@ -6,6 +6,7 @@ const {
 const { findRealEstatesForSearchRequest } = require("../helpers/db/realEstate");
 const { addMatches } = require("../helpers/db/searchRequestMatch");
 const { MAX_REAL_ESTATES_IN_FIRST_EMAIL } = require("../config/appConfig");
+const { EMAIL_FREQUENCY } = require("../common/enums");
 
 const matchRealEstates = async realEstates => {
   if (Array.isArray(realEstates)) {
@@ -18,18 +19,19 @@ const matchRealEstates = async realEstates => {
 
       searchRequestsPromise.then(searchRequests => {
         for (const searchRequest of searchRequests) {
-          const { id } = searchRequest;
+          const { id, emailFrequency } = searchRequest;
           if (!matches[id]) {
             matches[id] = {
               searchRequest,
-              realEstates: []
+              realEstates: [],
+              notifyNow: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
             };
           }
           matches[id].realEstates.push(realEstate);
           matchingRecords.push({
             searchRequestId: searchRequest.id,
             realEstateId: realEstate.id,
-            notified: false
+            notified: emailFrequency === EMAIL_FREQUENCY.ASAP.stringId
           });
         }
       });
@@ -62,7 +64,7 @@ const matchSearchRequest = async searchRequest => {
     matchingRecords.push({
       searchRequestId,
       realEstateId: realEstate.id,
-      notified: false
+      notified: true
     });
   }
 

app/views/goAgain.ejs

@@ -1,10 +1,30 @@
-<div class="row centered-element">
-  Super. Poslali smo Vam potvrdni email na Vašu email adresu.
-  Poslije tog emaila, svaki put kada nađemo nove nekretnine koje Vam odgovaraju
-  javićemo Vam emailom.
-  <br><br>
-   <a href="/" class="">
-      Nova pretraga
-    </a>
+<br>
+<div class="row left-align">
+    <p>Super. Poslali smo Vam potvrdni email na Vašu email adresu.
+        Poslije tog emaila, svaki put kada nađemo nove nekretnine koje Vam odgovaraju
+        javićemo Vam emailom.
+    </p>
+    <a href="/" class="">Nova pretraga</a>
+</div>
+<div class="row left-align">
+    <p>Ako koristite gmail sa uključenom opcijom <strong>Promotions</strong>
+        postoji šansa da nećete dobiti notifikaciju kada email od Kivija dođe.
+        Da biste počeli dobijati notifikacije otiđite u <strong>Promotions</strong> label i
+        pomjerite email u <strong>Primary</strong> label kao na videu ispod.
+    </p>
+
+    <br>
+    <div class="hide-on-med-and-down">
+        <h6>Web preglednik</h6>
+        <br>
+        <img class="full-width" src="assets/images/web-promotions.gif">
+        <br>
+        <h6>Mobilna Gmail aplikacija</h6>
+        <br>
+    </div>
+
+    <div class="col s12 m6 l6 push-m3 push-l3">
+        <img src="assets/images/android-promotions.gif" width="100%">
+    </div>
 </div>
 

app/views/layout.ejs

@@ -10,7 +10,6 @@
     gtag('config', '<%= process.env.GA_ID %>');
   </script>
 
-  <link href="https://fonts.googleapis.com/css?family=Roboto&display=swap" rel="stylesheet">
   <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
   <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/13.1.5/nouislider.min.css">
@@ -19,6 +18,7 @@
   <script src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
   <meta charset="UTF-8" />
   <link rel="stylesheet" href="/assets/main.css">
+  <link rel="stylesheet" href="/assets/segment.css">
 
   <link rel="apple-touch-icon" sizes="180x180" href="/assets/apple-touch-icon.png">
   <link rel="icon" type="image/png" sizes="32x32" href="/assets/favicon-32x32.png">
@@ -28,6 +28,20 @@
   <meta name="msapplication-TileColor" content="#da532c">
   <meta name="theme-color" content="#ffffff">
 
+  <meta name="title" content="Kivi.ba">
+  <meta name="description" content="Neka dom nađe vas">
+
+  <meta property="og:title" content="Kivi.ba">
+  <meta property="og:description" content="Neka dom nađe vas">
+  <meta property="og:image" content="http://www.kivi.ba/assets/images/thumbnail-big.png">
+  <meta property="og:image:width" content="1200">
+  <meta property="og:image:height" content="627">
+  <meta property="og:url" content="https://www.kivi.ba">
+  <meta name="twitter:card" content="summary_large_image">
+
+  <meta property="og:site_name" content="Kivi.ba">
+  <meta name="twitter:image:alt" content="Kivi.ba - Neka dom nađe vas">
+
   <%if (title) { %>
     <title> <%= title %> - Kivi.ba</title>
     <% } else { %>

app/views/location.ejs

@@ -42,7 +42,7 @@
   function locateMe() {
     if (navigator.geolocation) {
 
-      const onLocationSuccess = (position) => {
+      function onLocationSuccess (position) {
         const coordinates = position && position.coords ? position.coords : null;
         if (coordinates){
           const longitude = coordinates.longitude || null;
@@ -53,7 +53,7 @@
             map.setZoom(16);
           }
         }
-      };
+      }
 
       navigator.geolocation.getCurrentPosition(onLocationSuccess);
     }
@@ -162,8 +162,8 @@
     input.attachEvent = addEventListenerWrapper
   }
 
-  $(document).ready(() => {
-    $("#submit").click(() => {
+  $(document).ready(function() {
+    $("#submit").click(function() {
       const mapBounds = map.getBounds();
 
       $("#north").val(mapBounds.getNorthEast().lat());
@@ -178,4 +178,4 @@
   });
 </script>
 <script src="https://maps.googleapis.com/maps/api/js?key=AIzaSyAna8ohfV2HBMcxGk_29vqxU5Z_bDickqg&language=bs&libraries=places&callback=initMap" async
-  defer></script>
+  defer></script>

app/views/partials/range.ejs

@@ -1,53 +0,0 @@
-<form method="POST" id="form-range">
-
-  <div class="row center-align no-ui-slider centered-element-small" id="slider"></div>
-
-  <div class="col s6 push-s3 centered-element-small">
-    <a id="btnsubmit" href="#" class="next-center-button waves-effect waves-light btn">
-      Dalje
-    </a>
-  </div>
-
-  <input type="hidden" name="from" id="from" />
-  <input type="hidden" name="to" id="to" />
-</form>
-
-<script>
-
-  $(document).ready(() => {
-
-    var slider = document.getElementById('slider');
-
-    const unitFormat = wNumb({
-        decimals: 3,
-        thousand: '.',
-        suffix: '<%= unit %>'
-      })
-
-
-    noUiSlider.create(slider, {
-      start: [<%= rangeFrom.value %>, <%= rangeTo.value %>],
-      connect: true,
-      tooltips: true,
-      step: <%= rangeFrom.step %>,
-      range: {
-        'min': <%= rangeFrom.min %>,
-        'max': <%= rangeTo.max %>
-    },
-      format: unitFormat
-    });
-
-
-    $("#btnsubmit").click(() => {
-      const sliderValues = slider.noUiSlider.get();
-      
-      $("#from").val(unitFormat.from(sliderValues[0]));
-      $("#to").val(unitFormat.from(sliderValues[1]));
-
-      $("#form-range").submit();
-      // });
-
-    });
-  });
-
-</script>

app/views/queryReview.ejs

@@ -1,7 +1,8 @@
+<br><br>
 <form method="POST" id="form-queryreview">
     <div class="row center-align">
       <ul class="collection with-header">
-        <% for(const stepData of queryReviewData) { %>
+        <% for(const stepData of queryReviewTableData) { %>
           <li class="collection-item" >
             <div id="<%= stepData.id %>" ><%= stepData.title || '-' %>
               <a href="<%= stepData.url %>" class="kivi-color secondary-content">
@@ -12,6 +13,26 @@
         <% } %>
       </ul>
     </div>
+    <div class="row center-align">
+        <h6>Slanje obavještenja</h6>
+        <span class="segmented">
+                <label>
+                    <input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.ASAP.id %>"
+                    <% if (selectedEmailFrequency === EMAIL_FREQUENCY.ASAP.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= EMAIL_FREQUENCY.ASAP.title %></span>
+                </label>
+
+                <label>
+                    <input type="radio" name="emailFrequency" value="<%= EMAIL_FREQUENCY.DAILY.id %>"
+                    <% if (selectedEmailFrequency === EMAIL_FREQUENCY.DAILY.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= EMAIL_FREQUENCY.DAILY.title %></span>
+                </label>
+		    </span>
+    </div>
     <div class="row center-align">
         <div class="col">
             <input id="email" name="email" type="email" placeholder="vas.email@mail.com" <% if (email) { %>value="<%= email %>" <% } %> required size="250" />
@@ -46,9 +67,24 @@
 </form>
 
 <script>
- $(document).ready( () => {
-   $("#submit").click( () => {
-     $("#form-queryreview").submit();
+
+ $(document).ready(function() {
+   $("#submit").click(function() {
+       const simpleEmailRegex = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
+       const email = $("#email").val();
+       const confirmEmail = $("#confirmEmail").val();
+
+       if (email !== confirmEmail){
+           $("#error-label-email").text("Greška ! Unešeni emailovi nisu isti");
+           return;
+       }
+
+       if (simpleEmailRegex.test(email)){
+           $("#submit").attr("disabled", true);
+           $("#form-queryreview").submit();
+       }else{
+           $("#error-label-email").text("Greška ! Unešeni email nije validan");
+       }
    });
  });
 </script>

app/views/realEstateFilters.ejs

@@ -5,32 +5,57 @@
     <h5>Cijena</h5>
     <br><br>
     <div class="center-align no-ui-slider" id="priceFilter"></div>
-    <input type="hidden" id="priceFilterMin" name="priceFilterMin">
-    <input type="hidden" id="priceFilterMax" name="priceFilterMax">
   </div>
 
-  <br><br>
+  <br>
+  <div class="row">
+    <div class="col s5 m3 l3 push-m1 push-l2">
+      <input class="sliderInputBox" type="number" id="priceMin" name="priceMin">
+
+    </div>
+    <div class="col s5 m3 l3 push-s1 push-m4 push-l4">
+      <input class="sliderInputBox" type="number" id="priceMax" name="priceMax">
+    </div>
+  </div>
+
+  <br>
 
   <div class="row center-align">
     <h5>Površina</h5>
     <br><br>
     <div class="center-align no-ui-slider" id="sizeFilter"></div>
-    <input type="hidden" id="sizeFilterMin" name="sizeFilterMin">
-    <input type="hidden" id="sizeFilterMax" name="sizeFilterMax">
   </div>
 
-  <br><br>
+  <br>
+  <div class="row">
+    <div class="col s5 m3 l3 push-m1 push-l2">
+      <input class="sliderInputBox" type="number" id="sizeMin" name="sizeMin">
+
+    </div>
+    <div class="col s5 m3 l3 push-s1 push-m4 push-l4">
+      <input class="sliderInputBox" type="number" id="sizeMax" name="sizeMax">
+    </div>
+  </div>
+
+  <br>
 
   <% if(hasGardenSize) { %>
     <div class="row center-align">
       <h5>Površina okućnice</h5>
       <br><br>
       <div class="center-align no-ui-slider" id="gardenSizeFilter"></div>
-      <input type="hidden" id="gardenSizeFilterMin" name="gardenSizeFilterMin">
-      <input type="hidden" id="gardenSizeFilterMax" name="gardenSizeFilterMax">
     </div>
 
-    <br><br>
+    <br>
+    <div class="row">
+      <div class="col s5 m3 l3 push-m1 push-l2">
+        <input class="sliderInputBox" type="number" id="gardenSizeMin" name="gardenSizeMin">
+
+      </div>
+      <div class="col s5 m3 l3 push-s1 push-m4 push-l4">
+        <input class="sliderInputBox" type="number" id="gardenSizeMax" name="gardenSizeMax">
+      </div>
+    </div>
   <% } %>
 
   <div class="row">
@@ -41,43 +66,122 @@
 </form>
 
 <script>
-  $(document).ready(() => {
-    const priceFormat = wNumb({
-      thousand: ".",
-      suffix: " KM"
-    });
-    const sizeFormat = wNumb({
-      thousand: ".",
-      suffix: " m2"
-    });
+  $(document).ready(function() {
+    const priceSliderOptions = <%- priceSliderOptions %>;
+    const sizeSliderOptions = <%- sizeSliderOptions %>;
+    const priceStep = priceSliderOptions.step;
+    const sizeStep = sizeSliderOptions.step;
+    delete priceSliderOptions.step;
+    delete sizeSliderOptions.step;
+
+    function updatePriceInputs(values, handle, unencoded) {
+      $("#priceMin").val(Math.round(unencoded[0]/priceStep)*priceStep);
+      $("#priceMax").val(Math.round(unencoded[1]/priceStep)*priceStep);
+    }
+    function updateSizeInputs(values, handle, unencoded) {
+      $("#sizeMin").val(Math.round(unencoded[0]/sizeStep)*sizeStep);
+      $("#sizeMax").val(Math.round(unencoded[1]/sizeStep)*sizeStep);
+    }
 
     const priceSlider = document.getElementById("priceFilter");
-    const extendedPriceSliderOptions = {...<%- priceSliderOptions %>, format: priceFormat};
-    noUiSlider.create(priceSlider, extendedPriceSliderOptions);
-
     const sizeSlider = document.getElementById("sizeFilter");
-    const extendedSizeSliderOptions = {...<%- sizeSliderOptions %>, format: sizeFormat};
-    noUiSlider.create(sizeSlider, extendedSizeSliderOptions);
+
+    const priceSliderObject = noUiSlider.create(priceSlider, priceSliderOptions);
+    const sizeSliderObject = noUiSlider.create(sizeSlider, sizeSliderOptions);
+
+    priceSliderObject.on('slide', updatePriceInputs);
+    sizeSliderObject.on('slide', updateSizeInputs);
+
+    function priceMinChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const currentValues = priceSliderObject.get();
+        const newValue = element.currentTarget.value;
+        const fixedNewValue = newValue > currentValues[1] ? currentValues[1] : newValue;
+        priceSliderObject.set([fixedNewValue, null]);
+        $("#priceMin").val(Math.round(priceSliderObject.get()[0]));
+      }
+    }
+    function priceMaxChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const newValue = element.currentTarget.value;
+        priceSliderObject.set([null, newValue]);
+        $("#priceMax").val(Math.round(priceSliderObject.get()[1]));
+      }
+    }
+    $("#priceMin").val(priceSliderOptions.start[0]);
+    $("#priceMax").val(priceSliderOptions.start[1]);
+    $("#priceMin").change(priceMinChangeHandler);
+    $("#priceMax").change(priceMaxChangeHandler);
+
+    function sizeMinChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const currentValues = sizeSliderObject.get();
+        const newValue = element.currentTarget.value;
+        const fixedNewValue = newValue > currentValues[1] ? currentValues[1] : newValue;
+        sizeSliderObject.set([fixedNewValue, null]);
+        $("#sizeMin").val(Math.round(sizeSliderObject.get()[0]));
+      }
+    }
+    function sizeMaxChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const newValue = element.currentTarget.value;
+        sizeSliderObject.set([null, newValue]);
+        $("#sizeMax").val(Math.round(sizeSliderObject.get()[1]));
+      }
+    }
+    $("#sizeMin").val(sizeSliderOptions.start[0]);
+    $("#sizeMax").val(sizeSliderOptions.start[1]);
+    $("#sizeMin").change(sizeMinChangeHandler);
+    $("#sizeMax").change(sizeMaxChangeHandler);
 
     <% if(hasGardenSize) { %>
-      const gardenSizeSlider = document.getElementById("gardenSizeFilter");
-      const extendedGardenSizeSliderOptions = {...<%- gardenSizeSliderOptions %>, format: sizeFormat};
-      noUiSlider.create(gardenSizeSlider, extendedGardenSizeSliderOptions);
+    const gardenSizeSliderOptions = <%- gardenSizeSliderOptions %>;
+    const gardenSizeStep = gardenSizeSliderOptions.step;
+    delete gardenSizeSliderOptions.step;
+
+    function updateGardenSizeInputs(values, handle, unencoded) {
+      $("#gardenSizeMin").val(Math.round(unencoded[0]/gardenSizeStep)*gardenSizeStep);
+      $("#gardenSizeMax").val(Math.round(unencoded[1]/gardenSizeStep)*gardenSizeStep);
+    }
+
+    const gardenSizeSlider = document.getElementById("gardenSizeFilter");
+    const gardenSizeSliderObject = noUiSlider.create(gardenSizeSlider, gardenSizeSliderOptions);
+    gardenSizeSliderObject.on('slide', updateGardenSizeInputs);
+    function gardenSizeMinChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const currentValues = gardenSizeSliderObject.get();
+        const newValue = element.currentTarget.value;
+        const fixedNewValue = newValue > currentValues[1] ? currentValues[1] : newValue;
+        gardenSizeSliderObject.set([fixedNewValue, null]);
+        $("#gardenSizeMin").val(Math.round(gardenSizeSliderObject.get()[0]));
+      }
+    }
+    function gardenSizeMaxChangeHandler(element) {
+      if (element && element.currentTarget && element.currentTarget.value){
+        const newValue = element.currentTarget.value;
+        gardenSizeSliderObject.set([null, newValue]);
+        $("#gardenSizeMin").val(Math.round(gardenSizeSliderObject.get()[0]));
+      }
+    }
+    $("#gardenSizeMin").val(gardenSizeSliderOptions.start[0]);
+    $("#gardenSizeMax").val(gardenSizeSliderOptions.start[1]);
+    $("#gardenSizeMin").change("step", gardenSizeMinChangeHandler);
+    $("#gardenSizeMax").change("step", gardenSizeMaxChangeHandler);
     <% } %>
 
-    $("#submit").click(() => {
+    $("#submit").click(function() {
       const priceFilterValues = priceSlider.noUiSlider.get();
-      $("#priceFilterMin").val(priceFormat.from(priceFilterValues[0]));
-      $("#priceFilterMax").val(priceFormat.from(priceFilterValues[1]));
+      $("#priceFilterMin").val(priceFilterValues[0]);
+      $("#priceFilterMax").val(priceFilterValues[1]);
 
       const sizeFilterValues = sizeSlider.noUiSlider.get();
-      $("#sizeFilterMin").val(sizeFormat.from(sizeFilterValues[0]));
-      $("#sizeFilterMax").val(sizeFormat.from(sizeFilterValues[1]));
+      $("#sizeFilterMin").val(sizeFilterValues[0]);
+      $("#sizeFilterMax").val(sizeFilterValues[1]);
 
       <% if (hasGardenSize) { %>
         const gardenSizeFilterValues = gardenSizeSlider.noUiSlider.get();
-        $("#gardenSizeFilterMin").val(sizeFormat.from(gardenSizeFilterValues[0]));
-        $("#gardenSizeFilterMax").val(sizeFormat.from(gardenSizeFilterValues[1]));
+        $("#gardenSizeFilterMin").val(gardenSizeFilterValues[0]);
+        $("#gardenSizeFilterMax").val(gardenSizeFilterValues[1]);
       <% } %>
 
       $("#filtersForm").submit();

app/views/realEstateType.ejs

@@ -1,25 +1,50 @@
+<br><br>
 <form method="POST" id="form-real-estate-type">
-<div class="row center-align">
-  <div class="collection">
-    <% for(const realEstateType of realEstateTypes) { %>
+    <div class="center-align">
 
-      <a href="#" class="waves-effect collection-item"
-         style="color: #02adba"
-         id="<%= realEstateType.id %>"
-         onclick="saveAndSubmit(this.id)"
-      >
-        <%= realEstateType.title %>
-      </a>
+        <div class="row">
+            <span class="segmented">
+                <label>
+                    <input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_SALE.id %>"
+                    <% if (selectedAdType === AD_TYPE.AD_TYPE_SALE.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_SALE.title %></span>
+                </label>
 
-    <% } %>
-  </div>
-  <input type="hidden" name="realEstateType" id="realEstateType" />
-</div>
+                <label>
+                    <input type="radio" name="adType" value="<%= AD_TYPE.AD_TYPE_RENT.id %>"
+                    <% if (selectedAdType === AD_TYPE.AD_TYPE_RENT.id) { %>
+                    checked
+                    <% } %>>
+                    <span class="label"><%= AD_TYPE.AD_TYPE_RENT.title %></span>
+                </label>
+		    </span>
+        </div>
+
+        <br>
+        <div id="realEstateTypeSelection" class="collection">
+            <% for(const realEstateType of realEstateTypes) { %>
+
+                <a class="waves-effect row collection-item"
+                   id="<%= realEstateType.id %>"
+                   href="#"
+                   style="color: #02adba"
+                   onclick="saveAndSubmit(this.id)"
+                >
+                    <span class="center-align"><%= realEstateType.title %></span>
+                </a>
+
+            <% } %>
+        </div>
+        <input type="hidden" name="realEstateType" id="realEstateType" />
+    </div>
 </form>
 
 <script>
   function saveAndSubmit(id) {
    $("#realEstateType").val(id);
+   $("#realEstateTypeSelection > a").attr("onclick", "");
    $("#form-real-estate-type").submit();
   }
 </script>

app/views/redirect.ejs

@@ -20,7 +20,7 @@
     </h6>
 </div>
 <script>
-    window.onload = () => {
+    window.onload = function() {
         document.getElementById('realEstateUrl').click();
     }
 </script>

development.env

@@ -23,6 +23,7 @@ SOURCE_EMAIL=info@saburly.com
 #=============== CRAWLER SETTINGS===============#
 CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds
 STOP_CRAWLER=Non-zero value will skip crawler execution
+PRINT_CRAWLER_DEBUG_INFO=Non-zero value will print crawler debugging info to the server console
 #==OLX==
 OLX_MAX_PAGES=Restrict crawler to this number of pages
 OLX_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -30,3 +31,28 @@ OLX_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check commo
 OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
 OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
 OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+#==RENTAL==
+RENTAL_MAX_PAGES=Restrict crawler to this number of pages
+RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
+RENTAL_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+RENTAL_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+RENTAL_IGNORED_USERNAMES=!!! This is not used for rental crawler !!!
+RENTAL_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+RENTAL_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+#==PROSTOR==
+PROSTOR_MAX_PAGES=!!! This is not used for prostor crawler !!!
+PROSTOR_MAX_RESULTS_PER_PAGE=For Prostor crawler, this represents MAX RESULTS in total
+PROSTOR_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+PROSTOR_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
+PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
+PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+#==AKTIDO==
+AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
+AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
+AKTIDO_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
+AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
+AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

package.json

@@ -12,6 +12,7 @@
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
     "crawl": "cd app/crawler && node npmCrawl.js",
+    "daily-notify": "cd app/npmScripts && node npmDailyNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js"
   },

test/olxScrapeTest.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const olxCrawler = require("../app/crawler/specific/olx");
+const olxCrawler = require("../app/crawler/specificCrawlers/olx");
 
 const urlToScrape = process.argv[2] || undefined;