Changed sliders for different realestate types.

app/common/enums.js

@@ -7,7 +7,42 @@ const PRICE_SLIDER_OPTIONS_SALE = {
   step: 1000,
   connect: true
 };
-
+const FLAT_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [800000]
+  },
+  step: 5000,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [50000, 150000],
+  range: {
+    min: [0],
+    max: [1500000]
+  },
+  step: 10000,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [15000, 50000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 2000,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_SALE = {
+  start: [40000, 80000],
+  range: {
+    min: [0],
+    max: [2000000]
+  },
+  step: 10000,
+  connect: true
+};
 const PRICE_SLIDER_OPTIONS_RENT = {
   start: [300, 500],
   range: {
@@ -17,18 +52,62 @@ const PRICE_SLIDER_OPTIONS_RENT = {
   step: 50,
   connect: true
 };
-
+const FLAT_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [300, 600],
+  range: {
+    min: [0],
+    max: [4000]
+  },
+  step: 100,
+  connect: true
+};
+const HOUSE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [10000]
+  },
+  step: 100,
+  connect: true
+};
+const OFFICE_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [200, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
+const LAND_PRICE_SLIDER_OPTIONS_RENT = {
+  start: [500, 1000],
+  range: {
+    min: [0],
+    max: [20000]
+  },
+  step: 100,
+  connect: true
+};
 //This will be used for Flats, Apartments, Houses
 const HOME_SIZE_SLIDER_OPTIONS = {
   start: [30, 75],
   range: {
     min: [0],
-    max: [400]
+    max: [500]
   },
   step: 5,
   connect: true
 };
 
+const OFFICE_SIZE_SLIDER_OPTIONS = {
+  start: [30, 150],
+  range: {
+    min: [0],
+    max: [1200]
+  },
+  step: 10,
+  connect: true
+};
 const GARDEN_SIZE_SLIDER_OPTIONS = {
   start: [100, 1000],
   range: {
@@ -111,8 +190,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   HOUSE: {
@@ -126,8 +205,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   },
@@ -142,9 +221,9 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: OFFICE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: OFFICE_PRICE_SLIDER_OPTIONS_RENT,
-    sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
+    sizeSliderOptions: OFFICE_SIZE_SLIDER_OPTIONS
   },
   LAND: {
     id: "LAND",
@@ -157,8 +236,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: false,
     hasNumberOfFloors: false,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: LAND_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: LAND_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: LAND_SIZE_SLIDER_OPTIONS
   },
   APARTMENT: {
@@ -172,8 +251,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: false,
     hasFloorProp: true,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: FLAT_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: FLAT_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS
   },
   GARAGE: {
@@ -202,8 +281,8 @@ const AD_CATEGORY = {
     hasNumberOfRoom: true,
     hasNumberOfFloors: true,
     hasFloorProp: false,
-    priceSliderOptionsSale: PRICE_SLIDER_OPTIONS_SALE,
+    priceSliderOptionsSale: HOUSE_PRICE_SLIDER_OPTIONS_SALE,
-    priceSliderOptionsRent: PRICE_SLIDER_OPTIONS_RENT,
+    priceSliderOptionsRent: HOUSE_PRICE_SLIDER_OPTIONS_RENT,
     sizeSliderOptions: HOME_SIZE_SLIDER_OPTIONS,
     gardenSizeSliderOptions: GARDEN_SIZE_SLIDER_OPTIONS
   }
@@ -223,8 +302,7 @@ const AD_AGENCY = {
   OLX: "OLX",
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
-  AKTIDO: "AKTIDO",
+  AKTIDO: "AKTIDO"
-  SALJIC: "SALJIC"
 };
 
 const CRAWLER_AD_TYPE = {

app/crawler/crawl.js

@@ -9,14 +9,12 @@ const OlxCrawler = require("./specificCrawlers/olx");
 const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
-const SaljicCrawler = require("./specificCrawlers/saljic");
 
 const {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG,
+  AKTIDO_CONFIG
-  SALJIC_CONFIG
 } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
@@ -59,15 +57,6 @@ async function crawlAll() {
       AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
       AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
       AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
-    ),
-    new SaljicCrawler(
-      [postgresSaver],
-      SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
-      SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
-      SALJIC_CONFIG.SALJIC_MAX_PAGES,
-      SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
-      SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
-      SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
     )
   ];
 

app/crawler/crawlerConfig.js

@@ -5,12 +5,10 @@ const OLX_CONFIG = require("./specificConfigs/olx");
 const RENTAL_CONFIG = require("./specificConfigs/rental");
 const PROSTOR_CONFIG = require("./specificConfigs/prostor");
 const AKTIDO_CONFIG = require("./specificConfigs/aktido");
-const SALJIC_CONFIG = require("./specificConfigs/saljic");
 
 module.exports = {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG,
+  AKTIDO_CONFIG
-  SALJIC_CONFIG
 };

app/crawler/specificConfigs/saljic.js

@@ -1,34 +0,0 @@
-"use strict";
-const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
-
-const saljicCrawlerAdType =
-  process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
-    ? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
-    : null;
-
-const saljicParsedCrawlerAdCategories =
-  process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
-    ? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
-        category.trim()
-      )
-    : ["FLAT", "HOUSE"];
-
-const saljicIgnoredUsernames = [];
-
-const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
-  .map(categoryName =>
-    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
-  )
-  .filter(category => !!category);
-
-module.exports = {
-  SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
-  SALJIC_MAX_RESULTS_PER_PAGE:
-    parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
-  SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
-  SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
-  SALJIC_DELAY_BETWEEN_PAGES:
-    parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
-  SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
-};

app/crawler/specificCrawlers/saljic.js

@@ -1,611 +0,0 @@
-"use strict";
-
-const fetch = require("node-fetch");
-const cheerio = require("cheerio");
-const moment = require("moment-timezone");
-
-const {
-  AD_TYPE,
-  AD_CATEGORY,
-  AD_AGENCY,
-  AD_STATUS,
-  CRAWLER_AD_TYPE,
-  FURNISHING_TYPE,
-  HEATING_TYPE
-} = require("../../common/enums");
-
-const {
-  PRINT_CRAWLER_DEBUG,
-  DEFAULT_TIMEZONE
-} = require("../../config/appConfig");
-const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
-
-const SALJIC_ENUMS = {
-  SALJIC_AD_TYPE: {
-    [CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
-    [CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
-    [CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
-  },
-  SALJIC_AD_CATEGORY: {
-    [AD_CATEGORY.ALL.id]: "&input_kategorija=",
-    [AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
-    [AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
-    [AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
-    [AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
-    [AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
-    [AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
-    //[AD_CATEGORY.COTTAGE.id]: ""
-  }
-};
-
-class SaljicCrawler {
-  constructor(
-    savers = [],
-    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
-    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
-    maxPages = 5000,
-    maxResultsPerPage = 5000,
-    ignoredUsernames = [],
-    delayBetweenPages = 1000
-  ) {
-    this.savers = savers;
-    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
-    this.crawlerAdTypes = crawlerAdTypes;
-    this.crawlerAdCategories = crawlerAdCategories;
-    this.maxResultsPerPage = maxResultsPerPage;
-    this.delayBetweenPages = delayBetweenPages;
-  }
-
-  async crawl() {
-    const crawlAdCategories = this.crawlerAdCategories;
-
-    const newRealEstates = [];
-
-    if (crawlAdCategories) {
-      const indexGenerators = [];
-      for (const adCategory of crawlAdCategories) {
-        indexGenerators.push(this.categoryIndexer(adCategory));
-      }
-      //
-      //console.log(indexGenerators);
-      //
-      let done = false;
-      while (!done) {
-        const categoryIndexerPromises = [];
-        const generatorsToRemove = [];
-        for (const indexGenerator of indexGenerators) {
-          categoryIndexerPromises.push(indexGenerator.next());
-          generatorsToRemove.push(false);
-        }
-
-        const singlePageResults = await Promise.all(categoryIndexerPromises);
-        const entries = singlePageResults.entries();
-
-        for (const [index, { value: singlePageResult }] of entries) {
-          if (singlePageResult) {
-            const saveResults = await this.saveCrawledResults(singlePageResult);
-            const { newRecords } = saveResults;
-
-            newRealEstates.push(...newRecords);
-
-            if (
-              Array.isArray(newRecords) &&
-              newRecords.length === 0 &&
-              !SALJIC_FORCE_CRAWL
-            ) {
-              generatorsToRemove[index] = true;
-            }
-          } else {
-            //Generator returned undefined, remove this generator from array
-            generatorsToRemove[index] = true;
-            // console.log("Generator ", index + 1, "has no more pages");
-          }
-        }
-
-        // console.log("Generators state : ", generatorsToRemove);
-        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
-          if (generatorsToRemove[i]) {
-            // console.log("\tRemove generator ", i + 1);
-            indexGenerators.splice(i, 1);
-          }
-        }
-        if (indexGenerators.length === 0) {
-          done = true;
-        }
-
-        await this.sleep(this.delayBetweenPages);
-      }
-    }
-    return newRealEstates;
-  }
-
-  async *categoryIndexer(adCategory) {
-    let pageToIndex = 1;
-
-    const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
-    const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
-
-    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
-      while (true) {
-        const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
-        const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
-
-        const singlePageResults = await this.indexSinglePage(
-          urlPageToCrawl,
-          this.maxResultsPerPage
-        );
-
-        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
-          yield singlePageResults;
-        } else {
-          return undefined;
-        }
-
-        ++pageToIndex;
-        if (pageToIndex === this.maxPages) {
-          return undefined;
-        }
-      }
-    } else {
-      return undefined;
-    }
-  }
-
-  async indexSinglePage(url, maxResultsPerPage) {
-    if (PRINT_CRAWLER_DEBUG) {
-      console.log("[SALJIC] Index page : ", url);
-    }
-
-    try {
-      const res = await fetch(url);
-      const body = await res.text();
-      const $ = cheerio.load(body);
-      let hrefs = [];
-
-      $("#shop")
-        .find(".product")
-        .each((i, elem) => {
-          const href = $(elem)
-            .find("a")
-            .first()
-            .attr("href");
-          if (href) {
-            hrefs.push(href);
-          }
-        });
-
-      let adTypesTmp = [];
-
-      $("#shop")
-        .find(".product")
-        .each((i, elem) => {
-          const adType = $(elem)
-            .find(".trakica-search-page")
-            .text()
-            .trim();
-          if (adType) {
-            adTypesTmp.push(adType);
-          }
-        });
-
-      //Converting to AD_TYPE
-      const adTypes = adTypesTmp.map(adTypeText => {
-        return this.getAdTypeId(adTypeText);
-      });
-
-      //Converting to absolute URLs
-      const hrefsAbs = hrefs.map(link => {
-        return "https://www.saljicnekretnine.ba" + link;
-      });
-
-      let actualNoOfResults =
-        hrefsAbs.length <= maxResultsPerPage
-          ? hrefsAbs.length
-          : maxResultsPerPage;
-
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
-      }
-
-      const scrapedData = await Promise.all(asyncScraping);
-      const filteredScrapedData = scrapedData.filter(adData => !!adData);
-      return filteredScrapedData;
-    } catch (e) {
-      console.error("[SALJIC] Exception caught:" + e);
-      return [];
-    }
-  }
-
-  async scrapeAd(url, adType) {
-    console.log("[SALJIC] Scraping : ", url);
-    try {
-      const adPageSource = await fetch(url);
-      const body = await adPageSource.text();
-      const $ = cheerio.load(body);
-
-      // No information for status ex. PRODAN
-      const status = AD_STATUS.STATUS_NORMAL;
-      //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
-
-      //Extracting main properties
-      const propertySelectors = {
-        title:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
-        price:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
-        streetName:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
-
-        descriptions:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
-        latAndLong:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
-      };
-      const title = $(propertySelectors.title)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .replace(/ {1,}/g, " ")
-        .trim();
-
-      const priceText = $(propertySelectors.price)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .replace(/ {1,}/g, " ")
-        .trim();
-      const price =
-        priceText === "CIJENA NA UPIT"
-          ? null
-          : parseFloat(
-              priceText.substring(8, priceText.length - 3).replace(",", "")
-            );
-
-      const streetName = $(propertySelectors.streetName)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .trim();
-
-      const descriptions = $(propertySelectors.descriptions)
-        .text()
-        .trim();
-
-      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
-      const locationLat = parseFloat(latText) || null;
-      const locationLong = parseFloat(longText) || null;
-
-      //====== DETAIL INFORMATION FIELDS ==========
-      let area,
-        gardenSize,
-        numberOfRooms = null,
-        numberOfFloors = null,
-        floor = null,
-        accessRoadType = null,
-        heatingType = null,
-        furnishingType = null,
-        balcony = null,
-        newBuilding = null,
-        elevator = null,
-        water = null,
-        electricity = null,
-        drainageSystem = null,
-        registeredInZkBooks = null,
-        recentlyAdapted = null,
-        parking = null,
-        garage = null,
-        gas = null,
-        antiTheftDoor = null,
-        airCondition = null,
-        phoneConnection = null,
-        cableTV = null,
-        internet = null,
-        basementAttic = null,
-        storeRoom = null,
-        videoSurveillance = null,
-        alarm = null,
-        suitableForStudents = null,
-        includingBills = null,
-        animalsAllowed = null,
-        pool = null,
-        urbanPlanPermit = null,
-        buildingPermit = null,
-        utilityConnection = null,
-        distanceToRiver = null;
-      let publishedDate = null;
-      let renewedDate = null;
-      let realEstateType;
-      let numberOfViewsAgency = null;
-
-      //Extracting data - Glavne karakteristike
-      let mainFieldIndex = 1;
-      do {
-        const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
-
-        const mainField = $(mainFieldSelector)
-          .text()
-          .replace(/[\n\r\t]/gm, "")
-          .trim();
-
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
-        const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
-
-        switch (mainFieldTitle) {
-          case "Površina":
-            area = parseFloat(
-              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
-            );
-            break;
-          case "Okućnica":
-            gardenSize = parseFloat(
-              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
-            );
-            break;
-          case "Broj soba":
-            numberOfRooms = parseInt(mainFieldValue);
-            break;
-          case "Broj spratova":
-            numberOfFloors = parseInt(mainFieldValue);
-            break;
-          case "Sprat":
-            floor = parseInt(mainFieldValue);
-            break;
-          case "Godina renoviranja":
-            recentlyAdapted = true;
-            break;
-          case "Broj parking mjesta":
-            parking = true;
-            break;
-          case "Dostupno od":
-            const day = mainFieldValue.substring(0, 2);
-            const month = mainFieldValue.substring(3, 5);
-            const year = mainFieldValue.substring(6, mainFieldValue.length);
-            publishedDate = new Date(`${month}/${day}/${year}`);
-            break;
-          default:
-            break;
-        }
-
-        if (mainFieldTitle === "") {
-          break;
-        }
-        mainFieldIndex++;
-      } while (true);
-
-      //Extracting data - Sadrzaji
-      let additionalFieldIndex = 1;
-      do {
-        const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
-
-        const additionalField = $(additionalFieldSelector)
-          .text()
-          .trim();
-
-        if (additionalFieldIndex === 1) {
-          //Extracting data of real estate type
-          const categoryTmp = additionalField
-            .replace(/[\n\r\t]/gm, "")
-            .substring(
-              additionalField.indexOf("Kategorija") + 10,
-              additionalField.length
-            )
-            .trim();
-          realEstateType = this.getAdCategoryId(categoryTmp);
-        } else {
-          switch (additionalField) {
-            case "Internet":
-              internet = true;
-              break;
-            case "Garaža":
-              garage = true;
-              break;
-            case "Klima":
-              airCondition = true;
-              break;
-            case "Balkon":
-              balcony = true;
-              break;
-            case "Ostava":
-              storeRoom = true;
-              break;
-            case "Podrum":
-              basementAttic = true;
-              break;
-            case "Blindirana vrata":
-              antiTheftDoor = true;
-              break;
-            case "Voda":
-              water = true;
-              break;
-            case "Kablovska":
-              cableTV = true;
-              break;
-            case "Uknjiženo":
-              registeredInZkBooks = true;
-              break;
-            case "Grijanje - centralno":
-              heatingType = HEATING_TYPE.CENTRAL_CITY.id;
-              break;
-            case "Grijanje - plin":
-              heatingType = HEATING_TYPE.GAS.id;
-              break;
-            case "Grijanje - struja":
-              heatingType = HEATING_TYPE.ELECTRICITY.id;
-              break;
-            case "Grijanje":
-              heatingType = HEATING_TYPE.OTHER.id;
-              break;
-            case "Plin":
-              gas = true;
-              break;
-            case "Namješten":
-              furnishingType = FURNISHING_TYPE.FURNISHED.id;
-              break;
-            case "Alarm":
-              alarm = true;
-              break;
-            case "Video nadzor":
-              videoSurveillance = true;
-              break;
-            case "Lift":
-              elevator = true;
-              break;
-            case "Novogradnja":
-              newBuilding = true;
-              break;
-
-            default:
-              break;
-          }
-        }
-
-        if (additionalField === "") {
-          break;
-        }
-        additionalFieldIndex++;
-      } while (true);
-
-      //If no published date it takes current date of crawling
-      if (publishedDate) {
-        renewedDate = new Date();
-      } else {
-        publishedDate = new Date();
-        renewedDate = new Date();
-      }
-
-      const data = {
-        url,
-        agencyObjectId,
-        originAgencyName: AD_AGENCY.SALJIC,
-        realEstateType,
-        adType,
-        title,
-        price,
-        area,
-        gardenSize,
-        shortDescription: descriptions.substring(0, descriptions.indexOf(".")),
-        longDescription: descriptions,
-        streetNumber: 0,
-        streetName,
-        locality: "",
-        municipality: "",
-        city: "",
-        region: "",
-        entity: "",
-        country: "",
-        locationLat,
-        locationLong,
-        adStatus: status,
-        publishedDate,
-        renewedDate,
-        numberOfRooms,
-        numberOfFloors,
-        floor,
-        accessRoadType,
-        heatingType,
-        furnishingType,
-        balcony,
-        newBuilding,
-        elevator,
-        water,
-        electricity,
-        drainageSystem,
-        registeredInZkBooks,
-        recentlyAdapted,
-        parking,
-        garage,
-        gas,
-        antiTheftDoor,
-        airCondition,
-        phoneConnection,
-        cableTV,
-        internet,
-        basementAttic,
-        storeRoom,
-        videoSurveillance,
-        alarm,
-        suitableForStudents,
-        includingBills,
-        animalsAllowed,
-        pool,
-        urbanPlanPermit,
-        buildingPermit,
-        utilityConnection,
-        distanceToRiver,
-        numberOfViewsAgency
-      };
-      console.log(data);
-      return data;
-    } catch (e) {
-      console.error("Exception caught: " + e.message, "\r\nURL:", url);
-    }
-    return null;
-  }
-
-  //======= HELPER FUNCTIONS =============
-
-  getAdCategoryId(categoryText) {
-    switch (categoryText) {
-      case "Stan":
-        return AD_CATEGORY.FLAT.id;
-      case "Građevinsko zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Industrijsko zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Poljoprivredno zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Kuća":
-        return AD_CATEGORY.HOUSE.id;
-      case "Poslovni prostor":
-        return AD_CATEGORY.OFFICE.id;
-      case "Kancelarije":
-        return AD_CATEGORY.OFFICE.id;
-      case "Apartmani":
-        return AD_CATEGORY.APARTMENT.id;
-      case "Garaža":
-        return AD_CATEGORY.GARAGE.id;
-      case "Vikendica":
-        return AD_CATEGORY.COTTAGE.id;
-      default:
-        return undefined;
-    }
-  }
-
-  getAdTypeId(adTypeText) {
-    switch (adTypeText) {
-      case "PRODAJA":
-        return AD_TYPE.AD_TYPE_SALE.stringId;
-      case "NAJAM":
-        return AD_TYPE.AD_TYPE_RENT.stringId;
-      default:
-        return undefined;
-    }
-  }
-
-  async sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-
-  async saveCrawledResults(results) {
-    const savers = this.savers;
-
-    // for (const saver of savers) {
-    //   await saver.save(results);
-    // }
-
-    //For now, we use only Postgres saver, so ...
-    return savers[0].save(results);
-    //so that we can use some sequelize options and information when data is inserted
-  }
-}
-
-module.exports = SaljicCrawler;

development.env

@@ -59,8 +59,3 @@ AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to
 AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
-#==SALJIC NEKRETNINE==
-SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
-SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
-SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found