Renamed var.

Create fetch wrapper with user agent

See merge request saburly/marketalarm/web!98

app/common/enums.js

@@ -7,7 +7,42 @@ const PRICE_SLIDER_OPTIONS_SALE = {
   step: 1000,
   connect: true
 };
-
+const FLAT_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [800000]
+  },
+  step: 5000,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [1500000]
+  },
+  step: 10000,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [15000, 50000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 2000,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [40000, 80000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 10000,
+  connect: true
+};
 const PRICE_SLIDER_OPTIONS_RENT = {
   start: [300, 500],
   range: {
@@ -17,18 +52,62 @@ const PRICE_SLIDER_OPTIONS_RENT = {
   step: 50,
   connect: true
 };
-
+const FLAT_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 600],
+  range: {
+    min: [0],
+    max: [4000]
+  },
+  step: 100,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [10000]
+  },
+  step: 100,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [200, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
 //This will be used for Flats, Apartments, Houses
 const HOME_SIZE_SLIDER_OPTIONS = {
   start: [30, 75],
   range: {
     min: [0],
-    max: [400]
+    max: [500]
   },
   step: 5,
   connect: true
 };
 
+const OFFICE_SIZE_SLIDER_OPTIONS = {
+  start: [30, 150],
+  range: {
+    min: [0],
+    max: [1200]
+  },
+  step: 10,
+  connect: true
+};
 const GARDEN_SIZE_SLIDER_OPTIONS = {
   start: [100, 1000],
   range: {
@@ -111,8 +190,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   HOUSE: {
@@ -126,8 +205,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   },
@@ -142,9 +221,9 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT,
-    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+    sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS
   },
   LAND: {
     id: "LAND",
@@ -157,8 +236,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: false,
     hasNumberOfFloors: false,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
   },
   APARTMENT: {
@@ -172,8 +251,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   GARAGE: {
@@ -202,8 +281,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   }
@@ -216,14 +295,16 @@ const AD_STATUS = {
   STATUS_DELETED: 4,
   STATUS_URGENT: 5,
   STATUS_DISCOUNTED: 6,
-  STATUS_RENTED: 7
+  STATUS_RENTED: 7,
+  STATUS_VIP: 8
 };
 
 const AD_AGENCY = {
   OLX: "OLX",
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
-  AKTIDO: "AKTIDO"
+  AKTIDO: "AKTIDO",
+  SALJIC: "SALJIC"
 };
 
 const CRAWLER_AD_TYPE = {

app/config/appConfig.js

@@ -9,11 +9,15 @@ const APP_URL =
     ? process.env.APP_URL || "http://market-alarm"
     : process.env.APP_URL || `${APP_BASE_URL}:${APP_PORT}`;
 
+const STAGING = process.env.ENVIRONMENT !== "production";
+
 const DEFAULT_TIMEZONE = "Europe/Sarajevo";
 
 const CRAWLER_INTERVAL = parseInt(process.env.CRAWLER_INTERVAL) || 60;
 const STOP_CRAWLER = !!parseInt(process.env.STOP_CRAWLER);
 
+const CHECK_UP_DAYS = parseInt(process.env.CHECK_UP_DAYS) || 10;
+
 const AWS_EMAIL_CONFIG = {
   REGION: process.env.AWS_REGION || "",
   CREDENTIALS: {
@@ -32,6 +36,18 @@ const PRINT_CRAWLER_DEBUG = process.env.PRINT_CRAWLER_DEBUG_INFO || 0;
 
 const API_MAP_KEY = process.env.API_MAP_KEY || "";
 
+const PROSTOR_LOGIN = {
+  EMAIL: process.env.PROSTOR_LOGIN_EMAIL,
+  PASSWORD: process.env.PROSTOR_LOGIN_PASS
+};
+
+const USER_AGENT =
+  process.env.USER_AGENT ||
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
+
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -42,5 +58,11 @@ module.exports = {
   MAX_REAL_ESTATES_IN_EMAIL,
   MAX_REAL_ESTATES_IN_FIRST_EMAIL,
   PRINT_CRAWLER_DEBUG,
-  API_MAP_KEY
+  API_MAP_KEY,
+  STAGING,
+  CHECK_UP_DAYS,
+  PROSTOR_LOGIN,
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
 };

app/controllers/realEstates.js

@@ -2,13 +2,14 @@
 const {
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { AD_STATUS } = require("../common/enums");
 
 const getRealEstates = async (req, res) => {
   const searchRequestId = req.params["searchRequestId"] || "";
   const realEstates = await findRealEstatesForSearchRequest(searchRequestId);
 
   const title = "Nekretnine koje odgovaraju Vašim uslovima pretrage";
-  res.render("realEstates", { realEstates, title });
+  res.render("realEstates", { realEstates, title, AD_STATUS });
 };
 
 module.exports = {

app/controllers/redirect.js

@@ -1,9 +1,11 @@
 const { getRealEstateById } = require("../helpers/db/realEstate");
+const { AD_STATUS } = require("../common/enums");
 
 const getRedirect = async (req, res) => {
   const id = req.params.id || null;
   let error = false;
   let redirectUrl = undefined;
+  let vipAd = undefined;
   if (!id) {
     error = true;
   } else {
@@ -13,6 +15,7 @@ const getRedirect = async (req, res) => {
         error = true;
       } else {
         redirectUrl = realEstate.url;
+        vipAd = realEstate.adStatus === AD_STATUS.STATUS_VIP;
       }
     } catch (e) {
       error = true;
@@ -24,7 +27,7 @@ const getRedirect = async (req, res) => {
     res.render("notFound", { title });
   } else {
     const title = "Preusmjeravanje";
-    res.render("redirect", { title, redirectUrl });
+    res.render("redirect", { title, redirectUrl, vipAd });
   }
 };
 

app/crawler/crawl.js

@@ -9,12 +9,14 @@ const OlxCrawler = require("./specificCrawlers/olx");
 const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
+const SaljicCrawler = require("./specificCrawlers/saljic");
 
 const {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
 } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
@@ -57,6 +59,15 @@ async function crawlAll() {
       AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
       AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
       AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
+    ),
+    new SaljicCrawler(
+      [postgresSaver],
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
+      SALJIC_CONFIG.SALJIC_MAX_PAGES,
+      SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
+      SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
+      SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
     )
   ];
 

app/crawler/crawlerConfig.js

@@ -5,10 +5,12 @@ const OLX_CONFIG = require("./specificConfigs/olx");
 const RENTAL_CONFIG = require("./specificConfigs/rental");
 const PROSTOR_CONFIG = require("./specificConfigs/prostor");
 const AKTIDO_CONFIG = require("./specificConfigs/aktido");
+const SALJIC_CONFIG = require("./specificConfigs/saljic");
 
 module.exports = {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
 };

app/crawler/savers/postgres.js

@@ -1,6 +1,7 @@
 const moment = require("moment");
 
 const { bulkUpsertRealEstates } = require("../../helpers/db/realEstate");
+const { bulkUpsertPriceHistory } = require("../../helpers/db/priceHistory");
 
 class PostgresSaver {
   connect() {
@@ -11,6 +12,21 @@ class PostgresSaver {
 
   async save(results) {
     const savedRecords = await bulkUpsertRealEstates(results);
+    //Extruding data for price history table
+    const resultPrices = savedRecords.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+    const savedPrices = await bulkUpsertPriceHistory(resultPrices);
 
     if (Array.isArray(savedRecords)) {
       const newRealEstates = [];

app/crawler/specificConfigs/saljic.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const saljicCrawlerAdType =
+  process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
+    : null;
+
+const saljicParsedCrawlerAdCategories =
+  process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const saljicIgnoredUsernames = [];
+
+const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
+  SALJIC_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
+  SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
+  SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
+  SALJIC_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
+  SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
+};

app/crawler/specificCrawlers/aktido.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/olx.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");

app/crawler/specificCrawlers/prostor.js

@@ -1,8 +1,9 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
+const FormData = require("form-data");
 
 const {
   AD_TYPE,
@@ -16,7 +17,8 @@ const {
 
 const {
   PRINT_CRAWLER_DEBUG,
-  DEFAULT_TIMEZONE
+  DEFAULT_TIMEZONE,
+  PROSTOR_LOGIN
 } = require("../../config/appConfig");
 const { PROSTOR_FORCE_CRAWL } = require("../specificConfigs/prostor");
 
@@ -60,13 +62,16 @@ class ProstorCrawler {
 
   async crawl() {
     const crawlAdCategories = this.crawlerAdCategories;
-
+    //We need session cookie to use login privileges
+    const prostorCookie = await this.getCookies();
+    //New tag to check if crawler loged in
+    const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
     const newRealEstates = [];
-
+    //Crawl only if login was successful
-    if (crawlAdCategories) {
+    if (crawlAdCategories && login) {
       const indexGenerators = [];
       for (const adCategory of crawlAdCategories) {
-        indexGenerators.push(this.categoryIndexer(adCategory));
+        indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
       }
 
       let done = false;
@@ -119,13 +124,14 @@ class ProstorCrawler {
     return newRealEstates;
   }
 
-  async *categoryIndexer(adCategory) {
+  async *categoryIndexer(adCategory, prostorCookie) {
     const urlAdTypePart = PROSTOR_ENUMS.PROSTOR_AD_TYPE[this.crawlerAdTypes];
     const urlCategoryPart = PROSTOR_ENUMS.PROSTOR_AD_CATEGORY[adCategory];
     if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
       const urlPageToCrawl = `${this.baseUrl}?remove_sold=0${urlAdTypePart}${urlCategoryPart}`;
       const listOfAllRealEstates = await this.extractRealEstates(
-        urlPageToCrawl
+        urlPageToCrawl,
+        prostorCookie
       );
 
       let elementToStartIndexFrom = 0;
@@ -139,7 +145,8 @@ class ProstorCrawler {
           elementToStartIndexFrom += realEstatesForSinglePage.length;
 
           const singlePageResults = await this.indexSinglePage(
-            realEstatesForSinglePage
+            realEstatesForSinglePage,
+            prostorCookie
           );
 
           const filteredSinglePageResults = singlePageResults.filter(
@@ -163,10 +170,10 @@ class ProstorCrawler {
     }
   }
 
-  async indexSinglePage(realEstatesList) {
+  async indexSinglePage(realEstatesList, prostorCookie) {
     const asyncActions = [];
     for (const realEstate of realEstatesList) {
-      asyncActions.push(this.scrapeAd(realEstate));
+      asyncActions.push(this.scrapeAd(realEstate, prostorCookie));
     }
 
     try {
@@ -180,12 +187,19 @@ class ProstorCrawler {
     }
   }
 
-  async scrapeAd(realEstate) {
+  async scrapeAd(realEstate, prostorCookie) {
     const { lat, lng, property_name, price, size, link, status } = realEstate;
+
+    //Status information is given already in realestate list
+    const adStatus = ProstorCrawler.getStatusId(status);
+
     const url = `https://prostor.ba${link}`;
+
     // console.log("[PROSTOR] Scraping : ", url);
     try {
-      const adPageSource = await fetch(url);
+      const adPageSource = await fetch(url, {
+        headers: { Cookie: prostorCookie }
+      });
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
@@ -330,7 +344,6 @@ class ProstorCrawler {
         furnishingType = FURNISHING_TYPE.NOT_FURNISHED.id;
       }
 
-      const adStatus = ProstorCrawler.getStatusId(status);
       const title = property_name;
       const parsedPrice = parseFloat(price.replace(/\./g, "")) || null;
       const parsedArea = parseFloat(size);
@@ -408,13 +421,15 @@ class ProstorCrawler {
     }
   }
 
-  async extractRealEstates(url) {
+  async extractRealEstates(url, prostorCookie) {
     if (PRINT_CRAWLER_DEBUG) {
       console.log("[PROSTOR] Index page : ", url);
     }
 
     try {
-      const res = await fetch(url);
+      const res = await fetch(url, {
+        headers: { Cookie: prostorCookie }
+      });
       const body = await res.text();
       const $ = cheerio.load(body);
 
@@ -548,6 +563,8 @@ class ProstorCrawler {
         return AD_STATUS.STATUS_SOLD;
       case "Iznajmljeno":
         return AD_STATUS.STATUS_RENTED;
+      case "VIP ponuda":
+        return AD_STATUS.STATUS_VIP;
       default:
         console.log("[PROSTOR] Unknown AD_STATUS : [", statusText, "]");
         return AD_STATUS.STATUS_NORMAL;
@@ -569,6 +586,51 @@ class ProstorCrawler {
     return savers[0].save(results);
     //so that we can use some sequelize options and information when data is inserted
   }
+  async loginForScraping(PROSTOR_LOGIN, prostorCookie) {
+    let formData = new FormData();
+    formData.append("email", PROSTOR_LOGIN.EMAIL);
+    formData.append("password", PROSTOR_LOGIN.PASSWORD);
+
+    return fetch("https://prostor.ba/moj-prostor/prijava", {
+      method: "POST",
+      body: formData,
+      headers: { Cookie: prostorCookie }
+    })
+      .then(page => {
+        return page.text();
+      })
+      .then(resp => {
+        const $ = cheerio.load(resp);
+        if (
+          $("h1")
+            .text()
+            .indexOf("Dobrodošli") !== -1
+        ) {
+          console.log("[PROSTOR]: Crawler loged in!");
+          return true;
+        } else {
+          console.log("[PROSTOR]: Crawler login failed - wrong credentials!");
+          return false;
+        }
+      })
+      .catch(err => {
+        console.log("[PROSTOR]: Crawler login error ", err);
+      });
+  }
+  async getCookies() {
+    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
+      headers: { Cookie: "" }
+    });
+    const raw = getResponse.headers.raw()["set-cookie"];
+    const cookie = raw
+      .map(datastring => {
+        const data = datastring.split(";");
+        const cookieData = data[0];
+        return cookieData;
+      })
+      .join(";");
+    return cookie;
+  }
 }
 
 module.exports = ProstorCrawler;

app/crawler/specificCrawlers/rental.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const fetch = require("node-fetch");
+const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -399,7 +399,9 @@ class RentalCrawler {
         );
         if (!publishedDateMoment.isValid()) {
           throw {
-            message: `Invalid published date : ${extractedData["re_realEstates_inserted"]}`
+            message: `Invalid published date : ${
+              extractedData["re_realEstates_inserted"]
+            }`
           };
         }
 
@@ -410,7 +412,9 @@ class RentalCrawler {
         );
         if (!renewedDateMoment.isValid()) {
           throw {
-            message: `Invalid renewed date : ${extractedData["re_realEstates_edited"]}`
+            message: `Invalid renewed date : ${
+              extractedData["re_realEstates_edited"]
+            }`
           };
         }
 

app/crawler/specificCrawlers/saljic.js

@@ -0,0 +1,630 @@
+"use strict";
+
+const fetch = require("../../helpers/fetchWrapper");
+const cheerio = require("cheerio");
+const moment = require("moment-timezone");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  FURNISHING_TYPE,
+  HEATING_TYPE
+} = require("../../common/enums");
+
+const {
+  PRINT_CRAWLER_DEBUG,
+  DEFAULT_TIMEZONE
+} = require("../../config/appConfig");
+const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
+
+const SALJIC_ENUMS = {
+  SALJIC_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
+  },
+  SALJIC_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "&input_kategorija=",
+    [AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
+    [AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
+    [AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
+    [AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
+    [AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
+    [AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  }
+};
+
+class SaljicCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+      //
+      //console.log(indexGenerators);
+      //
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !SALJIC_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
+
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
+        const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
+
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[SALJIC] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let adTypesTmp = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const adType = $(elem)
+            .find(".trakica-search-page")
+            .text()
+            .trim();
+          if (adType) {
+            adTypesTmp.push(adType);
+          }
+        });
+
+      //Converting to AD_TYPE
+      const adTypes = adTypesTmp.map(adTypeText => {
+        return this.getAdTypeId(adTypeText);
+      });
+
+      //Converting to absolute URLs
+      const hrefsAbs = hrefs.map(link => {
+        return "https://www.saljicnekretnine.ba" + link;
+      });
+
+      let actualNoOfResults =
+        hrefsAbs.length <= maxResultsPerPage
+          ? hrefsAbs.length
+          : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[SALJIC] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url, adType) {
+    // console.log("[SALJIC] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      // No information for status ex. PRODAN
+      const status = AD_STATUS.STATUS_NORMAL;
+      //Extracting agency ID from url
+      const agencyObjectId = parseInt(url.substring(46, url.length));
+
+      //Extracting main properties
+      const propertySelectors = {
+        title:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
+        price:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
+        streetName:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
+
+        descriptions:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
+        latAndLong:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
+      };
+      const title = $(propertySelectors.title)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+
+      const priceText = $(propertySelectors.price)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+      const price =
+        priceText === "CIJENA NA UPIT"
+          ? null
+          : parseFloat(
+              priceText.substring(8, priceText.length - 3).replace(",", "")
+            );
+
+      const streetName = $(propertySelectors.streetName)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .trim();
+
+      const descriptions = $(propertySelectors.descriptions)
+        .text()
+        .replace(/\"/g, "")
+        .trim();
+
+      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
+      const latText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("marker=") + 7,
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
+      );
+      const longText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
+        latAndLongSrc.length
+      );
+      const locationLat = parseFloat(latText) || null;
+      const locationLong = parseFloat(longText) || null;
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area = null,
+        gardenSize = null,
+        numberOfRooms = null,
+        numberOfFloors = null,
+        floor = null,
+        accessRoadType = null,
+        heatingType = null,
+        furnishingType = null,
+        balcony = null,
+        newBuilding = null,
+        elevator = null,
+        water = null,
+        electricity = null,
+        drainageSystem = null,
+        registeredInZkBooks = null,
+        recentlyAdapted = null,
+        parking = null,
+        garage = null,
+        gas = null,
+        antiTheftDoor = null,
+        airCondition = null,
+        phoneConnection = null,
+        cableTV = null,
+        internet = null,
+        basementAttic = null,
+        storeRoom = null,
+        videoSurveillance = null,
+        alarm = null,
+        suitableForStudents = null,
+        includingBills = null,
+        animalsAllowed = null,
+        pool = null,
+        exchange = null,
+        urbanPlanPermit = null,
+        buildingPermit = null,
+        utilityConnection = null,
+        distanceToRiver = null;
+      let publishedDate = null;
+      let renewedDate = null;
+      let realEstateType;
+      let numberOfViewsAgency = null;
+      let numberOfViewsKivi = null;
+      let streetNumber = 0;
+      let adStatus = status;
+      let shortDescription = descriptions.substring(
+        0,
+        descriptions.indexOf(".")
+      );
+      let longDescription = descriptions;
+      //Extracting data - Glavne karakteristike
+      let mainFieldIndex = 1;
+      do {
+        const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
+
+        const mainField = $(mainFieldSelector)
+          .text()
+          .replace(/[\n\r\t]/gm, "")
+          .trim();
+
+        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldValue = mainField
+          .substring(mainField.indexOf("  "), mainField.length)
+          .trim();
+
+        switch (mainFieldTitle) {
+          case "Površina":
+            area = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Okućnica":
+            gardenSize = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Broj soba":
+            numberOfRooms = parseInt(mainFieldValue);
+            break;
+          case "Broj spratova":
+            numberOfFloors = parseInt(mainFieldValue);
+            break;
+          case "Sprat":
+            floor = parseInt(mainFieldValue);
+            break;
+          case "Godina renoviranja":
+            recentlyAdapted = true;
+            break;
+          case "Broj parking mjesta":
+            parking = true;
+            break;
+          case "Dostupno od":
+            const day = mainFieldValue.substring(0, 2);
+            const month = mainFieldValue.substring(3, 5);
+            const year = mainFieldValue.substring(6, mainFieldValue.length);
+            publishedDate = new Date(`${month}/${day}/${year}`);
+            break;
+          default:
+            break;
+        }
+
+        if (mainFieldTitle === "") {
+          break;
+        }
+        mainFieldIndex++;
+      } while (true);
+
+      //Extracting data - Sadrzaji
+      let additionalFieldIndex = 1;
+      do {
+        const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
+
+        const additionalField = $(additionalFieldSelector)
+          .text()
+          .trim();
+
+        if (additionalFieldIndex === 1) {
+          //Extracting data of real estate type
+          const categoryTmp = additionalField
+            .replace(/[\n\r\t]/gm, "")
+            .substring(
+              additionalField.indexOf("Kategorija") + 10,
+              additionalField.length
+            )
+            .trim();
+          realEstateType = this.getAdCategoryId(categoryTmp);
+        } else {
+          switch (additionalField) {
+            case "Internet":
+              internet = true;
+              break;
+            case "Garaža":
+              garage = true;
+              break;
+            case "Klima":
+              airCondition = true;
+              break;
+            case "Balkon":
+              balcony = true;
+              break;
+            case "Ostava":
+              storeRoom = true;
+              break;
+            case "Podrum":
+              basementAttic = true;
+              break;
+            case "Blindirana vrata":
+              antiTheftDoor = true;
+              break;
+            case "Voda":
+              water = true;
+              break;
+            case "Kablovska":
+              cableTV = true;
+              break;
+            case "Uknjiženo":
+              registeredInZkBooks = true;
+              break;
+            case "Grijanje - centralno":
+              heatingType = HEATING_TYPE.CENTRAL_CITY.id;
+              break;
+            case "Grijanje - plin":
+              heatingType = HEATING_TYPE.GAS.id;
+              break;
+            case "Grijanje - struja":
+              heatingType = HEATING_TYPE.ELECTRICITY.id;
+              break;
+            case "Grijanje":
+              heatingType = HEATING_TYPE.OTHER.id;
+              break;
+            case "Plin":
+              gas = true;
+              break;
+            case "Namješten":
+              furnishingType = FURNISHING_TYPE.FURNISHED.id;
+              break;
+            case "Alarm":
+              alarm = true;
+              break;
+            case "Video nadzor":
+              videoSurveillance = true;
+              break;
+            case "Lift":
+              elevator = true;
+              break;
+            case "Novogradnja":
+              newBuilding = true;
+              break;
+
+            default:
+              break;
+          }
+        }
+
+        if (additionalField === "") {
+          break;
+        }
+        additionalFieldIndex++;
+      } while (true);
+
+      //If no published date it takes current date of crawling
+      if (publishedDate) {
+        renewedDate = new Date();
+      } else {
+        publishedDate = new Date();
+        renewedDate = new Date();
+      }
+
+      const originAgencyName = AD_AGENCY.SALJIC;
+      const locality = "";
+      const municipality = "";
+      const city = "";
+      const region = "";
+      const entity = "";
+      const country = "";
+
+      const data = {
+        url,
+        agencyObjectId,
+        originAgencyName,
+        realEstateType,
+        adType,
+        title,
+        price,
+        area,
+        gardenSize,
+        shortDescription,
+        longDescription,
+        streetNumber,
+        streetName,
+        locality,
+        municipality,
+        city,
+        region,
+        entity,
+        country,
+        locationLat,
+        locationLong,
+        adStatus,
+        publishedDate,
+        renewedDate,
+        numberOfRooms,
+        numberOfFloors,
+        floor,
+        accessRoadType,
+        heatingType,
+        furnishingType,
+        balcony,
+        newBuilding,
+        elevator,
+        water,
+        electricity,
+        drainageSystem,
+        registeredInZkBooks,
+        recentlyAdapted,
+        parking,
+        garage,
+        gas,
+        antiTheftDoor,
+        airCondition,
+        phoneConnection,
+        cableTV,
+        internet,
+        basementAttic,
+        storeRoom,
+        videoSurveillance,
+        alarm,
+        suitableForStudents,
+        includingBills,
+        animalsAllowed,
+        pool,
+        exchange,
+        urbanPlanPermit,
+        buildingPermit,
+        utilityConnection,
+        distanceToRiver,
+        numberOfViewsAgency,
+        numberOfViewsKivi
+      };
+
+      return data;
+    } catch (e) {
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "Stan":
+        return AD_CATEGORY.FLAT.id;
+      case "Građevinsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Industrijsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Poljoprivredno zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Kuća":
+        return AD_CATEGORY.HOUSE.id;
+      case "Poslovni prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "Kancelarije":
+        return AD_CATEGORY.OFFICE.id;
+      case "Apartmani":
+        return AD_CATEGORY.APARTMENT.id;
+      case "Garaža":
+        return AD_CATEGORY.GARAGE.id;
+      case "Vikendica":
+        return AD_CATEGORY.COTTAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "PRODAJA":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "NAJAM":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = SaljicCrawler;

app/helpers/db/priceHistory.js

@@ -0,0 +1,20 @@
+"use strict";
+const db = require("../../models/index");
+const sequelize = require("sequelize");
+
+const bulkUpsertPriceHistory = async priceHistoryData => {
+  try {
+    const order = [["realEstateId", "desc"]];
+
+    return await db.PriceHistory.bulkCreate(priceHistoryData, {
+      order,
+      ignoreDuplicates: true
+    });
+  } catch (e) {
+    console.log("Error bulk upserting priceHistory : ", e);
+  }
+};
+
+module.exports = {
+  bulkUpsertPriceHistory
+};

app/helpers/db/realEstate.js

@@ -332,10 +332,14 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
+  //When includeIncompleteAds are not defined - null it will consider it true
   const order = [["updatedAt", "desc"]];
 
   return db.RealEstate.findAll({
-    where: includeIncompleteAds ? queryIncludeIncomplete : query,
+    where:
+      includeIncompleteAds || includeIncompleteAds == null
+        ? queryIncludeIncomplete
+        : query,
     limit: maxResults,
     order
   });

app/helpers/db/searchRequest.js

@@ -157,7 +157,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
     } else {
       // If real estate dont have defined number of rooms ex. null
       //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
+      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
       numberOfRoomsQuery = {
         [Op.or]: [
           {
@@ -176,7 +176,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -226,7 +229,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -275,7 +281,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
           },
           {
             includeIncompleteAds: {
-              [Op.eq]: true
+              [Op.or]: {
+                [Op.eq]: true,
+                [Op.is]: null
+              }
             }
           }
         ]
@@ -313,7 +322,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -347,7 +359,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -381,7 +396,10 @@ const findSearchRequestsForRealEstate = async realEstate => {
               },
               {
                 includeIncompleteAds: {
-                  [Op.eq]: true
+                  [Op.or]: {
+                    [Op.eq]: true,
+                    [Op.is]: null
+                  }
                 }
               }
             ]
@@ -423,10 +441,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
       [Op.eq]: "ANY"
     };
   }
-  //Tag to check if incomplete ads are accepted in query
+  //Tag to check if incomplete ads are accepted in query which is default
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
-      [Op.eq]: true
+      [Op.or]: {
+        [Op.eq]: true,
+        [Op.is]: null
+      }
     };
   }
 

app/helpers/db/searchRequestMatch.js

@@ -1,5 +1,8 @@
 "use strict";
 const db = require("../../models/index");
+const sequelize = require("sequelize");
+const Op = sequelize.Op;
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -40,6 +43,42 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
+const findAllRequestsForCheckUp = async () => {
+  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
+  //The ones that received notification for real estate CHECK_UP_DAYS days from now
+  const date = new Date();
+  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
+  date.setDate(checkUpDate);
+  const dateQuery = {
+    createdAt: {
+      [Op.gte]: date
+    }
+  };
+
+  const excludedMatches = await db.SearchRequestMatch.findAll({
+    attributes: ["searchRequestId"],
+    where: dateQuery,
+    order: [["searchRequestId", "ASC"]]
+  });
+
+  const excludedRequestsAll = excludedMatches.map(match => {
+    return match.dataValues.searchRequestId;
+  });
+  //Removing duplicate search request id-s for optimization
+  const excludedRequests = [...new Set(excludedRequestsAll)];
+
+  const query = {
+    subscribed: true,
+    id: {
+      [Op.notIn]: excludedRequests
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: query
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -50,5 +89,6 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp
 };

app/helpers/emailContentGenerator.js

@@ -1,10 +1,18 @@
 "use strict";
 
-const { MAX_REAL_ESTATES_IN_EMAIL, APP_URL } = require("../config/appConfig");
+const {
-const { AD_CATEGORY } = require("../common/enums");
+  MAX_REAL_ESTATES_IN_EMAIL,
+  APP_URL,
+  STAGING
+} = require("../config/appConfig");
+const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");
 
-const generateEmailFooter = searchRequestId => {
+//Tag to recognize staging from development
-  return `<div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
+const stagingTag = STAGING ? "[STAGING] " : "";
+
+const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
+  return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>
+    <div>Ako želite prestati dobijati obavještenja za ovu pretragu, <a href="${APP_URL}/odjava/${searchRequestId}">odjavite ovdje</a></div>
      <div>Ako želite pogledati ili promijeniti uslove za ovu pretragu, <a href="${APP_URL}/pregled/${searchRequestId}">pogledajte ovdje</a></div>
      <br/> 
      <strong>Vaš,<br/>Kivi tim</strong>`;
@@ -23,17 +31,24 @@ const generateRealEstateLinks = realEstates => {
 const generateNotificationEmail = (
   realEstates,
   searchRequestId,
+  noAllRealEstates,
   dailyNotification = false
 ) => {
   const truncateList = realEstates.length > MAX_REAL_ESTATES_IN_EMAIL;
+
   const realEstatesToShow = truncateList
     ? realEstates.slice(0, MAX_REAL_ESTATES_IN_EMAIL)
     : realEstates;
 
   const allRealEstatesLink = `${APP_URL}/nekretnine/${searchRequestId}`;
+
+  const emailFrequencyTitle = dailyNotification
+    ? EMAIL_FREQUENCY.DAILY.title
+    : EMAIL_FREQUENCY.ASAP.title;
+
   const realEstateLinks = generateRealEstateLinks(realEstatesToShow);
-  const moreRealEstates = `<div>Kompletan spisak nekretnina možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
+  const moreRealEstates = `<div>Kompletan spisak nekretnina (${noAllRealEstates}) možete pogledati na <a href="${allRealEstatesLink}">listi nekretnina</a><div>`;
-  const emailFooter = generateEmailFooter(searchRequestId);
+  const emailFooter = generateEmailFooter(searchRequestId, emailFrequencyTitle);
   const asapMessageBody =
     realEstates.length > 1
       ? "Pronašli smo nekretnine koje odgovaraju Vašoj pretrazi"
@@ -46,7 +61,7 @@ const generateNotificationEmail = (
 
   const messageBody = dailyNotification ? dailyMessageBody : asapMessageBody;
 
-  return `<h3>Zdravo</h3>
+  return `<h3>${stagingTag}Zdravo</h3>
   <h4>${messageBody}</h4>
   <div>
   ${realEstateLinks}
@@ -59,6 +74,28 @@ const generateNotificationEmail = (
 
 const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
   const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  let adTypeTitle = "";
+  switch (searchRequest.adType) {
+    case AD_TYPE.AD_TYPE_SALE.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_SALE.title;
+      break;
+    case AD_TYPE.AD_TYPE_RENT.stringId:
+      adTypeTitle = AD_TYPE.AD_TYPE_RENT.title;
+      break;
+    default:
+      adTypeTitle = "-";
+      break;
+  }
+  let emailFrequencyTitle;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.ASAP.title;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.DAILY.title;
+      break;
+  }
+
   const {
     id,
     gardenSizeMin,
@@ -70,6 +107,7 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
   } = searchRequest;
 
   const realEstateLinks = generateRealEstateLinks(matchingRealEstates);
+
   const instantRealEstatesText = `<br/>
        <div>
         U međuvremenu pogledajte neke od nedavno objavljenih nekretnina koje odgovaraju Vašim uslovima pretrage :<br/>
@@ -80,13 +118,14 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
     ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
     : ``;
 
-  const emailFooter = generateEmailFooter(id);
+  const emailFooter = generateEmailFooter(id, emailFrequencyTitle);
 
-  return `<h3>Zdravo</h3>
+  return `<h3>${stagingTag}Zdravo</h3>
   <div>Naručili ste da Vam javimo ako se nekretnina sa navedenim uslovima pojavi u oglasima:</div>
   <br/>
   <div>
     <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Vrsta oglasa: </strong>${adTypeTitle}</div>
     <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
     ${gardenSize}
     <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
@@ -98,7 +137,7 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
 
 const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
   if (numberOfRealEstates === 1) {
-    return `Kivi: ${singleRealEstateTitle}`;
+    return `${stagingTag}Kivi: ${singleRealEstateTitle}`;
   }
 
   const leastSignificantDigit = numberOfRealEstates % 10;
@@ -106,7 +145,7 @@ const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
   const secondLeastSignificantDigit = numberWithoutLastDigit % 10;
 
   if (leastSignificantDigit === 1 && secondLeastSignificantDigit !== 1) {
-    return `Kivi : ${numberOfRealEstates} nova nekretnina`;
+    return `${stagingTag}Kivi : ${numberOfRealEstates} nova nekretnina`;
   }
 
   if (
@@ -114,14 +153,58 @@ const generateEmailSubject = (numberOfRealEstates, singleRealEstateTitle) => {
     leastSignificantDigit <= 4 &&
     secondLeastSignificantDigit !== 1
   ) {
-    return `Kivi: ${numberOfRealEstates} nove nekretnine`;
+    return `${stagingTag}Kivi: ${numberOfRealEstates} nove nekretnine`;
   }
 
-  return `Kivi: ${numberOfRealEstates} novih nekretnina`;
+  return `${stagingTag}Kivi: ${numberOfRealEstates} novih nekretnina`;
+};
+
+const generateCheckUpEmail = searchRequest => {
+  const realEstateType = AD_CATEGORY[searchRequest.realEstateType];
+  const {
+    id,
+    gardenSizeMin,
+    gardenSizeMax,
+    sizeMin,
+    sizeMax,
+    priceMin,
+    priceMax
+  } = searchRequest;
+
+  let emailFrequencyTitle;
+  switch (searchRequest.emailFrequency) {
+    case EMAIL_FREQUENCY.ASAP.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.ASAP.title;
+      break;
+    case EMAIL_FREQUENCY.DAILY.stringId:
+      emailFrequencyTitle = EMAIL_FREQUENCY.DAILY.title;
+      break;
+  }
+
+  const gardenSize = realEstateType.hasGardenSize
+    ? `<div><strong>Kvadratura okućnice: Od ${gardenSizeMin} do ${gardenSizeMax} m2</strong></div>`
+    : ``;
+
+  const emailFooter = generateEmailFooter(id, emailFrequencyTitle);
+
+  return `<h3>${stagingTag}Zdravo</h3>
+  <div><strong>Kivi tim traži nekretnine za Vas i kada to ne vidite.</strong></div>
+  <br />
+  <div>Vaša trenutno aktivna pretraga je:</div>
+  <br/>
+  <div>
+    <div><strong>Tip nekretnine: </strong>${realEstateType.title}</div>
+    <div><strong>Kvadratura nekretnine:</strong> Od ${sizeMin} do ${sizeMax} m2</div>
+    ${gardenSize}
+    <div><strong>Cijena:</strong> ${priceMin} do ${priceMax} KM</div>
+  </div>
+  <br/>
+  ${emailFooter}`;
 };
 
 module.exports = {
   generateNotificationEmail,
   generateNewSearchRequestEmail,
-  generateEmailSubject
+  generateEmailSubject,
+  generateCheckUpEmail
 };

app/helpers/fetchWrapper.js

@@ -0,0 +1,21 @@
+const nodeFetch = require("node-fetch");
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
+} = require("../config/appConfig");
+
+const fetch = async (url, options = {}) => {
+  const newOptions = Object.assign({}, options);
+  if (!newOptions["headers"]) {
+    newOptions["headers"] = {};
+  }
+  newOptions["headers"]["User-Agent"] = USER_AGENT;
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    : url;
+
+  return nodeFetch(urlAdaptedForScraping, newOptions);
+};
+
+module.exports = fetch;

app/migrations/20200121000524-add-priceHistory-table.js

@@ -0,0 +1,42 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    const tableFields = {
+      id: {
+        type: Sequelize.BIGINT,
+        autoIncrement: true,
+        allowNull: false,
+        primaryKey: true
+      },
+      realEstateId: {
+        type: Sequelize.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: Sequelize.REAL,
+        unique: "uniquePriceRealEstate"
+      },
+      createdAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      },
+      updatedAt: {
+        type: Sequelize.DATE,
+        defaultValue: Sequelize.literal("NOW()")
+      }
+    };
+    return queryInterface.createTable("PriceHistory", tableFields);
+  },
+
+  down: queryInterface => {
+    return queryInterface.dropTable("PriceHistory", {});
+  }
+};

app/migrations/20200121094500-add-constraint-priceHistory.js

@@ -0,0 +1,10 @@
+"use strict";
+module.exports = {
+  up: (queryInterface, Sequelize) =>
+    queryInterface.addConstraint("PriceHistory", ["realEstateId", "price"], {
+      type: "unique",
+      name: "uniquePriceRealEstate"
+    }),
+  down: queryInterface =>
+    queryInterface.removeConstraint("PriceHistory", "uniquePriceRealEstate")
+};

app/models/priceHistory.js

@@ -0,0 +1,44 @@
+"use strict";
+
+module.exports = (sequalize, DataTypes) => {
+  const PriceHistory = sequalize.define(
+    "PriceHistory",
+    {
+      id: {
+        type: DataTypes.BIGINT,
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false
+      },
+      realEstateId: {
+        type: DataTypes.BIGINT,
+        allowNull: false,
+        unique: "uniquePriceRealEstate",
+        references: {
+          model: "RealEstates",
+          key: "id"
+        },
+        onUpdate: "CASCADE",
+        onDelete: "SET NULL"
+      },
+      price: {
+        type: DataTypes.REAL,
+        unique: "uniquePriceRealEstate"
+      }
+    },
+    {
+      freezeTableName: true
+    }
+  );
+
+  PriceHistory.associate = models => {
+    PriceHistory.hasMany(models.RealEstate, {
+      foreignKey: "id",
+      sourceKey: "realEstateId",
+      targetKey: "id",
+      as: "realEstates"
+    });
+  };
+
+  return PriceHistory;
+};

app/models/searchRequest.js

@@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
-          [
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0]
-          ]
-        ],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },

app/npmScripts/npmCheckUpNotify.js

@@ -0,0 +1,6 @@
+"use strict";
+const { checkUpNotify } = require("../services/notificationService");
+//For testing pursposes
+(async () => {
+  await checkUpNotify();
+})();

app/public/main.css

@@ -154,3 +154,7 @@ h3 {
   margin-top: 2rem;
   margin-bottom: 1rem;
 }
+
+.estates-link {
+  color: rgba(0, 0, 0, 0.87);
+}

app/seeders/20200121150443-initial-price-history.js

@@ -0,0 +1,34 @@
+"use strict";
+
+const { RealEstate } = require("../models");
+
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    //Reading initial data from RealEstate table in db
+    const realEstateInitialData = await RealEstate.findAll();
+    //Extruding data for table PriceHistory
+    const priceHistoryInitialData = realEstateInitialData.map(realEstate => {
+      //Null values canot be recognized by ignore duplicates in sequalize
+      //Value price = 0 indicates 'cijena na upit'
+      const priceTmp =
+        realEstate.dataValues.price === null ? 0 : realEstate.dataValues.price;
+
+      return {
+        realEstateId: realEstate.dataValues.id,
+        price: priceTmp,
+        createdAt: realEstate.dataValues.createdAt,
+        updatedAt: realEstate.dataValues.updatedAt
+      };
+    });
+
+    return queryInterface.bulkInsert(
+      "PriceHistory",
+      priceHistoryInitialData,
+      {}
+    );
+  },
+
+  async down(queryInterface, Sequelize) {
+    return queryInterface.bulkDelete("PriceHistory", null, {});
+  }
+};

app/services/notificationService.js

@@ -1,4 +1,8 @@
 "use strict";
+const { STAGING } = require("../config/appConfig");
+
+const stagingTag = STAGING ? "[STAGING] " : "";
+
 const {
   matchRealEstates,
   matchSearchRequest
@@ -6,9 +10,14 @@ const {
 const {
   generateNotificationEmail,
   generateNewSearchRequestEmail,
-  generateEmailSubject
+  generateEmailSubject,
+  generateCheckUpEmail
 } = require("../helpers/emailContentGenerator");
-const { findNotNotifiedMatches } = require("../helpers/db/searchRequestMatch");
+const {
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp,
+  findRealEstatesForSearchRequest
+} = require("../helpers/db/searchRequestMatch");
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -21,13 +30,17 @@ const notifyForNewSearchRequest = async searchRequest => {
 
   const searchRequestId = searchRequest.id;
   const matchingRealEstates = matches[searchRequestId].realEstates;
-
   const emailContent = generateNewSearchRequestEmail(
     searchRequest,
     matchingRealEstates
   );
   const { email } = searchRequest;
-  await sendEmail(email, "Kivi - novi zahtjev za pretragu", emailContent);
+
+  await sendEmail(
+    email,
+    `${stagingTag} Kivi - novi zahtjev za pretragu`,
+    emailContent
+  );
 };
 
 const notifyMatches = async (matches, dailyNotification = false) => {
@@ -39,10 +52,18 @@ const notifyMatches = async (matches, dailyNotification = false) => {
     const { email, subscribed } = searchRequest;
     if (notifyNow && subscribed) {
       const allMatchingRealEstates = matches[id].realEstates || [];
+
+      //Variable allMatchingRealEstates are real estates that are "new" on the market
+      //the ones that we notify user in this moment, not all that already exists in db
+      //New variable allRealEstates are all real estates that exists in db for search req
+      const allRealEstates = await findRealEstatesForSearchRequest(id);
+      const noAllRealEstates = allRealEstates.length;
+
       if (allMatchingRealEstates.length > 0) {
         const emailContent = generateNotificationEmail(
           allMatchingRealEstates,
           id,
+          noAllRealEstates,
           dailyNotification
         );
         const emailSubject = generateEmailSubject(
@@ -109,8 +130,26 @@ const notifyRequestsWithDailyOption = async () => {
   await notifyMatches(matches, true);
 };
 
+const checkUpNotify = async () => {
+  /* const searchRequestsForCheckUp = await findAllRequestsForCheckUp();
+
+  const asyncSendEmailActions = [];
+
+  for (const searchRequest of searchRequestsForCheckUp) {
+    const { email } = searchRequest.dataValues;
+    const emailSubject = `${stagingTag}Kivi: Mi tražimo nekretnine za vas!`;
+    const emailContent = generateCheckUpEmail(searchRequest.dataValues);
+
+    const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
+    asyncSendEmailActions.push(sendEmailPromise);
+    sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+  }
+  await Promise.all(asyncSendEmailActions); */
+};
+
 module.exports = {
   notifyForNewRealEstates,
   notifyForNewSearchRequest,
-  notifyRequestsWithDailyOption
+  notifyRequestsWithDailyOption,
+  checkUpNotify
 };

app/views/advancedFilters.ejs

@@ -61,9 +61,8 @@
 <p class="distinguished">
   <label class="checkbox-label">
     <input type="checkbox" class="filled-in" name="includeIncompleteAds"
-      <% if (includeIncompleteAds) { %>
            checked
-      <% } %>>
+      >
     <span>Uključi i oglase bez potpunih informacija</span>
   </label>
 </p>

app/views/realEstates.ejs

@@ -1,13 +1,29 @@
 <div class="row center-align">
-    <ul class="collection with-header">
+  <ul class="collection with-header">
-      <% for(const realEstate of realEstates) { %>
+    <% for(const realEstate of realEstates) { %>
-      <li class="collection-item">
+    <li class="collection-item">
-        <div><%= realEstate.title %>
+      <% if(realEstate.adStatus === AD_STATUS.STATUS_VIP) {%>
-          <a href="<%= realEstate.url %>" class="kivi-color secondary-content">
+      <div>
+        <% //This needs to do redirecting instead of direct link to realestate
+        %>
+        <a href="/redirect/<%= realEstate.id %>" class="estates-link">
+          <%= realEstate.title %>
+          <div class="kivi-color secondary-content">
             <i class="material-icons">send</i>
-          </a>
+          </div>
-        </div>
+        </a>
-      </li>
+      </div>
-      <% } %>
+      <%} else { %>
-    </ul>
+      <div>
-  </div>
+        <a href="<%= realEstate.url %>" class="estates-link">
+          <%= realEstate.title %>
+          <div class="kivi-color secondary-content">
+            <i class="material-icons">send</i>
+          </div>
+        </a>
+      </div>
+      <% }%>
+    </li>
+    <% } %>
+  </ul>
+</div>

app/views/redirect.ejs

@@ -1,26 +1,49 @@
-<br><br>
+<br /><br />
 <div class="center">
-    <div class="preloader-wrapper big active center">
+  <div class="preloader-wrapper big active center">
-        <div class="kivi-spinner-color spinner-layer spinner-green-only">
+    <div class="kivi-spinner-color spinner-layer spinner-green-only">
-            <div class="circle-clipper left">
+      <div class="circle-clipper left">
-                <div class="circle"></div>
+        <div class="circle"></div>
-            </div><div class="gap-patch">
+      </div>
-                <div class="circle"></div>
+      <div class="gap-patch">
-            </div><div class="circle-clipper right">
+        <div class="circle"></div>
-                <div class="circle"></div>
+      </div>
-            </div>
+      <div class="circle-clipper right">
-        </div>
+        <div class="circle"></div>
+      </div>
     </div>
+  </div>
 </div>
 
-<br>
+<br />
+<% if(vipAd) { %>
 <div class="center">
-    <h6>
+  <h6>
-        <a href="<%= redirectUrl %>" rel="noreferrer" id="realEstateUrl">Kliknite ovdje ako Vas web preglednik ne preusmjeri automatski</a>
+    Ovaj oglas zahtijeva da budete član
-    </h6>
+    <a href="https://prostor.ba/" rel="noreferrer">Prostor.ba</a>.
+    <br />
+    <br />
+    <a href="https://prostor.ba/moj-prostor/prijava" rel="noreferrer"
+      >Ulogujte se</a
+    >
+    ili napravite
+    <a href="https://prostor.ba/moj-prostor/registracija" rel="noreferrer"
+      >novi račun</a
+    >, a potom otvorite <a href="<%= redirectUrl %>" rel="noreferrer">oglas</a>.
+  </h6>
 </div>
+<% } else { %>
+<div class="center">
+  <h6>
+    <a href="<%= redirectUrl %>" rel="noreferrer" id="realEstateUrl"
+      >Kliknite ovdje ako Vas web preglednik ne preusmjeri automatski</a
+    >
+  </h6>
+</div>
+<% }%>
+
 <script>
-    window.onload = function() {
+  window.onload = function() {
-        document.getElementById('realEstateUrl').click();
+    document.getElementById("realEstateUrl").click();
-    }
+  };
 </script>

development.env

@@ -8,15 +8,24 @@ SEQUELIZE_LOGGING=0- no sequelize logging, 1- log to the console
 PORT=Port for the app, defaults to 5000
 APP_BASE_URL=base url for the app
 
+ENVIRONMENT=Variable to denote development, staging and production
+
+USER_AGENT=User agent header to send in fetch requests
+
 MAX_REAL_ESTATES_IN_EMAIL=Max number of real estates that will be shown in email, others will be truncated and URL with full list will be shwon
 MAX_REAL_ESTATES_IN_FIRST_EMAIL=Max number of real estates that will be shown in first (welcome) email
 
+CHECK_UP_DAYS=Check up email is sent after this number of days without notification
 #=============== GOOGLE ANALYTICS =============#
 GA_ID=Google Analytics ID
 
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api 
+
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)
@@ -51,6 +60,8 @@ PROSTOR_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories t
 PROSTOR_IGNORED_USERNAMES=!!! This is not used for prostor crawler !!!
 PROSTOR_DELAY_BETWEEN_PAGES=!!! This is not used for prostor crawler !!!
 PROSTOR_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+PROSTOR_LOGIN_EMAIL=Email of valid Prostor.ba account for crawling purposes
+PROSTOR_LOGIN_PASS=Password of valid Prostor.ba account for crawling purposes
 #==AKTIDO==
 AKTIDO_MAX_PAGES=Restrict crawler to this number of pages
 AKTIDO_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -59,3 +70,8 @@ AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to
 AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+#==SALJIC NEKRETNINE==
+SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
+SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found

index.js

@@ -12,6 +12,7 @@ const {
 } = require("./app/config/appConfig");
 const routes = require("./app/routes");
 const { crawlAll } = require("./app/crawler/crawl");
+const { checkUpNotify } = require("./app/services/notificationService");
 const {
   notifyForNewRealEstates
 } = require("./app/services/notificationService");
@@ -45,4 +46,7 @@ const crawl = () => {
     });
   }
 };
+
 setInterval(crawl, CRAWLER_INTERVAL * 1000);
+
+setInterval(checkUpNotify, 1000 * 60 * 60 * 24);

package-lock.json

@@ -1346,13 +1346,23 @@
       "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
     },
     "form-data": {
-      "version": "2.3.3",
+      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
-      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+      "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
       "requires": {
         "asynckit": "^0.4.0",
-        "combined-stream": "^1.0.6",
+        "combined-stream": "^1.0.8",
         "mime-types": "^2.1.12"
+      },
+      "dependencies": {
+        "combined-stream": {
+          "version": "1.0.8",
+          "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+          "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+          "requires": {
+            "delayed-stream": "~1.0.0"
+          }
+        }
       }
     },
     "forwarded": {
@@ -3430,6 +3440,18 @@
         "tough-cookie": "~2.4.3",
         "tunnel-agent": "^0.6.0",
         "uuid": "^3.3.2"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "2.3.3",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+          "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.6",
+            "mime-types": "^2.1.12"
+          }
+        }
       }
     },
     "require-directory": {

package.json

@@ -8,11 +8,13 @@
     "start": "node ./index.js",
     "start-mon": "nodemon ./index.js",
     "migrate": "cd app && npx sequelize db:migrate",
+    "seed": "cd app && npx sequelize db:seed:all",
     "setup": "docker build -t marketalerts .  && docker run -e POSTGRES_USER=docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=marketalerts --name pg_marketalerts -d -p 5432:5432 marketalerts && sleep 10 && npm run migrate",
     "docker-start": "docker start pg_marketalerts",
     "docker-stop": "docker stop pg_marketalerts",
     "crawl": "cd app/crawler && node npmCrawl.js",
     "daily-notify": "cd app/npmScripts && node npmDailyNotify.js",
+    "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js",
     "test-rental-scraper": "cd test && node rentalScrapeTest.js"
@@ -39,6 +41,7 @@
     "express": "^4.16.4",
     "express-ejs-layouts": "^2.5.0",
     "express-layout": "^0.1.0",
+    "form-data": "^3.0.0",
     "html-to-text": "^5.1.1",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.26",