Changed default to switched on.

.prettierignore

@@ -0,0 +1 @@
+*.ejs

app/common/enums.js

@@ -223,8 +223,7 @@ const AD_AGENCY = {
   OLX: "OLX",
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
-  AKTIDO: "AKTIDO",
-  SALJIC: "SALJIC"
+  AKTIDO: "AKTIDO"
 };
 
 const CRAWLER_AD_TYPE = {

app/controllers/realEstateFilters.js

@@ -35,7 +35,8 @@ const getFilters = async (req, res) => {
     balcony,
     elevator,
     newBuilding,
-    accessRoadType
+    accessRoadType,
+    includeWithoutPrice
   } = searchRequest;
   const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
 
@@ -115,7 +116,8 @@ const getFilters = async (req, res) => {
     advancedSegmentSelectFilterValues,
     advancedRangeFilterObjects,
     advancedRangeFilterValues,
-    includeIncompleteAds
+    includeIncompleteAds,
+    includeWithoutPrice
   });
 };
 
@@ -191,6 +193,7 @@ const postFilters = async (req, res) => {
   });
 
   const includeIncompleteAds = req.body.includeIncompleteAds === "on";
+  const includeWithoutPrice = req.body.includeWithoutPrice === "on";
 
   const balcony = req.body.balcony === "on";
   const elevator = req.body.elevator === "on";
@@ -217,6 +220,7 @@ const postFilters = async (req, res) => {
   searchRequest.newBuilding = newBuilding;
 
   searchRequest.includeIncompleteAds = includeIncompleteAds;
+  searchRequest.includeWithoutPrice = includeWithoutPrice;
 
   searchRequest.accessRoadType = accessRoadType;
 

app/crawler/crawl.js

@@ -9,14 +9,12 @@ const OlxCrawler = require("./specificCrawlers/olx");
 const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
-const SaljicCrawler = require("./specificCrawlers/saljic");
 
 const {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG,
-  SALJIC_CONFIG
+  AKTIDO_CONFIG
 } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
@@ -59,15 +57,6 @@ async function crawlAll() {
       AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
       AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
       AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
-    ),
-    new SaljicCrawler(
-      [postgresSaver],
-      SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
-      SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
-      SALJIC_CONFIG.SALJIC_MAX_PAGES,
-      SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
-      SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
-      SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
     )
   ];
 

app/crawler/crawlerConfig.js

@@ -5,12 +5,10 @@ const OLX_CONFIG = require("./specificConfigs/olx");
 const RENTAL_CONFIG = require("./specificConfigs/rental");
 const PROSTOR_CONFIG = require("./specificConfigs/prostor");
 const AKTIDO_CONFIG = require("./specificConfigs/aktido");
-const SALJIC_CONFIG = require("./specificConfigs/saljic");
 
 module.exports = {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG,
-  SALJIC_CONFIG
+  AKTIDO_CONFIG
 };

app/crawler/specificConfigs/saljic.js

@@ -1,34 +0,0 @@
-"use strict";
-const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
-
-const saljicCrawlerAdType =
-  process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
-    ? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
-    : null;
-
-const saljicParsedCrawlerAdCategories =
-  process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
-    ? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
-        category.trim()
-      )
-    : ["FLAT", "HOUSE"];
-
-const saljicIgnoredUsernames = [];
-
-const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
-  .map(categoryName =>
-    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
-  )
-  .filter(category => !!category);
-
-module.exports = {
-  SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
-  SALJIC_MAX_RESULTS_PER_PAGE:
-    parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
-  SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
-  SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
-  SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
-  SALJIC_DELAY_BETWEEN_PAGES:
-    parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
-  SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
-};

app/crawler/specificCrawlers/saljic.js

@@ -1,611 +0,0 @@
-"use strict";
-
-const fetch = require("node-fetch");
-const cheerio = require("cheerio");
-const moment = require("moment-timezone");
-
-const {
-  AD_TYPE,
-  AD_CATEGORY,
-  AD_AGENCY,
-  AD_STATUS,
-  CRAWLER_AD_TYPE,
-  FURNISHING_TYPE,
-  HEATING_TYPE
-} = require("../../common/enums");
-
-const {
-  PRINT_CRAWLER_DEBUG,
-  DEFAULT_TIMEZONE
-} = require("../../config/appConfig");
-const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
-
-const SALJIC_ENUMS = {
-  SALJIC_AD_TYPE: {
-    [CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
-    [CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
-    [CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
-  },
-  SALJIC_AD_CATEGORY: {
-    [AD_CATEGORY.ALL.id]: "&input_kategorija=",
-    [AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
-    [AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
-    [AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
-    [AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
-    [AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
-    [AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
-    //[AD_CATEGORY.COTTAGE.id]: ""
-  }
-};
-
-class SaljicCrawler {
-  constructor(
-    savers = [],
-    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
-    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
-    maxPages = 5000,
-    maxResultsPerPage = 5000,
-    ignoredUsernames = [],
-    delayBetweenPages = 1000
-  ) {
-    this.savers = savers;
-    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
-    this.crawlerAdTypes = crawlerAdTypes;
-    this.crawlerAdCategories = crawlerAdCategories;
-    this.maxResultsPerPage = maxResultsPerPage;
-    this.delayBetweenPages = delayBetweenPages;
-  }
-
-  async crawl() {
-    const crawlAdCategories = this.crawlerAdCategories;
-
-    const newRealEstates = [];
-
-    if (crawlAdCategories) {
-      const indexGenerators = [];
-      for (const adCategory of crawlAdCategories) {
-        indexGenerators.push(this.categoryIndexer(adCategory));
-      }
-      //
-      //console.log(indexGenerators);
-      //
-      let done = false;
-      while (!done) {
-        const categoryIndexerPromises = [];
-        const generatorsToRemove = [];
-        for (const indexGenerator of indexGenerators) {
-          categoryIndexerPromises.push(indexGenerator.next());
-          generatorsToRemove.push(false);
-        }
-
-        const singlePageResults = await Promise.all(categoryIndexerPromises);
-        const entries = singlePageResults.entries();
-
-        for (const [index, { value: singlePageResult }] of entries) {
-          if (singlePageResult) {
-            const saveResults = await this.saveCrawledResults(singlePageResult);
-            const { newRecords } = saveResults;
-
-            newRealEstates.push(...newRecords);
-
-            if (
-              Array.isArray(newRecords) &&
-              newRecords.length === 0 &&
-              !SALJIC_FORCE_CRAWL
-            ) {
-              generatorsToRemove[index] = true;
-            }
-          } else {
-            //Generator returned undefined, remove this generator from array
-            generatorsToRemove[index] = true;
-            // console.log("Generator ", index + 1, "has no more pages");
-          }
-        }
-
-        // console.log("Generators state : ", generatorsToRemove);
-        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
-          if (generatorsToRemove[i]) {
-            // console.log("\tRemove generator ", i + 1);
-            indexGenerators.splice(i, 1);
-          }
-        }
-        if (indexGenerators.length === 0) {
-          done = true;
-        }
-
-        await this.sleep(this.delayBetweenPages);
-      }
-    }
-    return newRealEstates;
-  }
-
-  async *categoryIndexer(adCategory) {
-    let pageToIndex = 1;
-
-    const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
-    const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
-
-    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
-      while (true) {
-        const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
-        const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
-
-        const singlePageResults = await this.indexSinglePage(
-          urlPageToCrawl,
-          this.maxResultsPerPage
-        );
-
-        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
-          yield singlePageResults;
-        } else {
-          return undefined;
-        }
-
-        ++pageToIndex;
-        if (pageToIndex === this.maxPages) {
-          return undefined;
-        }
-      }
-    } else {
-      return undefined;
-    }
-  }
-
-  async indexSinglePage(url, maxResultsPerPage) {
-    if (PRINT_CRAWLER_DEBUG) {
-      console.log("[SALJIC] Index page : ", url);
-    }
-
-    try {
-      const res = await fetch(url);
-      const body = await res.text();
-      const $ = cheerio.load(body);
-      let hrefs = [];
-
-      $("#shop")
-        .find(".product")
-        .each((i, elem) => {
-          const href = $(elem)
-            .find("a")
-            .first()
-            .attr("href");
-          if (href) {
-            hrefs.push(href);
-          }
-        });
-
-      let adTypesTmp = [];
-
-      $("#shop")
-        .find(".product")
-        .each((i, elem) => {
-          const adType = $(elem)
-            .find(".trakica-search-page")
-            .text()
-            .trim();
-          if (adType) {
-            adTypesTmp.push(adType);
-          }
-        });
-
-      //Converting to AD_TYPE
-      const adTypes = adTypesTmp.map(adTypeText => {
-        return this.getAdTypeId(adTypeText);
-      });
-
-      //Converting to absolute URLs
-      const hrefsAbs = hrefs.map(link => {
-        return "https://www.saljicnekretnine.ba" + link;
-      });
-
-      let actualNoOfResults =
-        hrefsAbs.length <= maxResultsPerPage
-          ? hrefsAbs.length
-          : maxResultsPerPage;
-
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
-      }
-
-      const scrapedData = await Promise.all(asyncScraping);
-      const filteredScrapedData = scrapedData.filter(adData => !!adData);
-      return filteredScrapedData;
-    } catch (e) {
-      console.error("[SALJIC] Exception caught:" + e);
-      return [];
-    }
-  }
-
-  async scrapeAd(url, adType) {
-    console.log("[SALJIC] Scraping : ", url);
-    try {
-      const adPageSource = await fetch(url);
-      const body = await adPageSource.text();
-      const $ = cheerio.load(body);
-
-      // No information for status ex. PRODAN
-      const status = AD_STATUS.STATUS_NORMAL;
-      //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
-
-      //Extracting main properties
-      const propertySelectors = {
-        title:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
-        price:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
-        streetName:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
-
-        descriptions:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
-        latAndLong:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
-      };
-      const title = $(propertySelectors.title)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .replace(/ {1,}/g, " ")
-        .trim();
-
-      const priceText = $(propertySelectors.price)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .replace(/ {1,}/g, " ")
-        .trim();
-      const price =
-        priceText === "CIJENA NA UPIT"
-          ? null
-          : parseFloat(
-              priceText.substring(8, priceText.length - 3).replace(",", "")
-            );
-
-      const streetName = $(propertySelectors.streetName)
-        .text()
-        .replace(/(\r\n|\n|\r)/gm, "")
-        .trim();
-
-      const descriptions = $(propertySelectors.descriptions)
-        .text()
-        .trim();
-
-      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
-      const locationLat = parseFloat(latText) || null;
-      const locationLong = parseFloat(longText) || null;
-
-      //====== DETAIL INFORMATION FIELDS ==========
-      let area,
-        gardenSize,
-        numberOfRooms = null,
-        numberOfFloors = null,
-        floor = null,
-        accessRoadType = null,
-        heatingType = null,
-        furnishingType = null,
-        balcony = null,
-        newBuilding = null,
-        elevator = null,
-        water = null,
-        electricity = null,
-        drainageSystem = null,
-        registeredInZkBooks = null,
-        recentlyAdapted = null,
-        parking = null,
-        garage = null,
-        gas = null,
-        antiTheftDoor = null,
-        airCondition = null,
-        phoneConnection = null,
-        cableTV = null,
-        internet = null,
-        basementAttic = null,
-        storeRoom = null,
-        videoSurveillance = null,
-        alarm = null,
-        suitableForStudents = null,
-        includingBills = null,
-        animalsAllowed = null,
-        pool = null,
-        urbanPlanPermit = null,
-        buildingPermit = null,
-        utilityConnection = null,
-        distanceToRiver = null;
-      let publishedDate = null;
-      let renewedDate = null;
-      let realEstateType;
-      let numberOfViewsAgency = null;
-
-      //Extracting data - Glavne karakteristike
-      let mainFieldIndex = 1;
-      do {
-        const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
-
-        const mainField = $(mainFieldSelector)
-          .text()
-          .replace(/[\n\r\t]/gm, "")
-          .trim();
-
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
-        const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
-
-        switch (mainFieldTitle) {
-          case "Površina":
-            area = parseFloat(
-              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
-            );
-            break;
-          case "Okućnica":
-            gardenSize = parseFloat(
-              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
-            );
-            break;
-          case "Broj soba":
-            numberOfRooms = parseInt(mainFieldValue);
-            break;
-          case "Broj spratova":
-            numberOfFloors = parseInt(mainFieldValue);
-            break;
-          case "Sprat":
-            floor = parseInt(mainFieldValue);
-            break;
-          case "Godina renoviranja":
-            recentlyAdapted = true;
-            break;
-          case "Broj parking mjesta":
-            parking = true;
-            break;
-          case "Dostupno od":
-            const day = mainFieldValue.substring(0, 2);
-            const month = mainFieldValue.substring(3, 5);
-            const year = mainFieldValue.substring(6, mainFieldValue.length);
-            publishedDate = new Date(`${month}/${day}/${year}`);
-            break;
-          default:
-            break;
-        }
-
-        if (mainFieldTitle === "") {
-          break;
-        }
-        mainFieldIndex++;
-      } while (true);
-
-      //Extracting data - Sadrzaji
-      let additionalFieldIndex = 1;
-      do {
-        const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
-
-        const additionalField = $(additionalFieldSelector)
-          .text()
-          .trim();
-
-        if (additionalFieldIndex === 1) {
-          //Extracting data of real estate type
-          const categoryTmp = additionalField
-            .replace(/[\n\r\t]/gm, "")
-            .substring(
-              additionalField.indexOf("Kategorija") + 10,
-              additionalField.length
-            )
-            .trim();
-          realEstateType = this.getAdCategoryId(categoryTmp);
-        } else {
-          switch (additionalField) {
-            case "Internet":
-              internet = true;
-              break;
-            case "Garaža":
-              garage = true;
-              break;
-            case "Klima":
-              airCondition = true;
-              break;
-            case "Balkon":
-              balcony = true;
-              break;
-            case "Ostava":
-              storeRoom = true;
-              break;
-            case "Podrum":
-              basementAttic = true;
-              break;
-            case "Blindirana vrata":
-              antiTheftDoor = true;
-              break;
-            case "Voda":
-              water = true;
-              break;
-            case "Kablovska":
-              cableTV = true;
-              break;
-            case "Uknjiženo":
-              registeredInZkBooks = true;
-              break;
-            case "Grijanje - centralno":
-              heatingType = HEATING_TYPE.CENTRAL_CITY.id;
-              break;
-            case "Grijanje - plin":
-              heatingType = HEATING_TYPE.GAS.id;
-              break;
-            case "Grijanje - struja":
-              heatingType = HEATING_TYPE.ELECTRICITY.id;
-              break;
-            case "Grijanje":
-              heatingType = HEATING_TYPE.OTHER.id;
-              break;
-            case "Plin":
-              gas = true;
-              break;
-            case "Namješten":
-              furnishingType = FURNISHING_TYPE.FURNISHED.id;
-              break;
-            case "Alarm":
-              alarm = true;
-              break;
-            case "Video nadzor":
-              videoSurveillance = true;
-              break;
-            case "Lift":
-              elevator = true;
-              break;
-            case "Novogradnja":
-              newBuilding = true;
-              break;
-
-            default:
-              break;
-          }
-        }
-
-        if (additionalField === "") {
-          break;
-        }
-        additionalFieldIndex++;
-      } while (true);
-
-      //If no published date it takes current date of crawling
-      if (publishedDate) {
-        renewedDate = new Date();
-      } else {
-        publishedDate = new Date();
-        renewedDate = new Date();
-      }
-
-      const data = {
-        url,
-        agencyObjectId,
-        originAgencyName: AD_AGENCY.SALJIC,
-        realEstateType,
-        adType,
-        title,
-        price,
-        area,
-        gardenSize,
-        shortDescription: descriptions.substring(0, descriptions.indexOf(".")),
-        longDescription: descriptions,
-        streetNumber: 0,
-        streetName,
-        locality: "",
-        municipality: "",
-        city: "",
-        region: "",
-        entity: "",
-        country: "",
-        locationLat,
-        locationLong,
-        adStatus: status,
-        publishedDate,
-        renewedDate,
-        numberOfRooms,
-        numberOfFloors,
-        floor,
-        accessRoadType,
-        heatingType,
-        furnishingType,
-        balcony,
-        newBuilding,
-        elevator,
-        water,
-        electricity,
-        drainageSystem,
-        registeredInZkBooks,
-        recentlyAdapted,
-        parking,
-        garage,
-        gas,
-        antiTheftDoor,
-        airCondition,
-        phoneConnection,
-        cableTV,
-        internet,
-        basementAttic,
-        storeRoom,
-        videoSurveillance,
-        alarm,
-        suitableForStudents,
-        includingBills,
-        animalsAllowed,
-        pool,
-        urbanPlanPermit,
-        buildingPermit,
-        utilityConnection,
-        distanceToRiver,
-        numberOfViewsAgency
-      };
-      console.log(data);
-      return data;
-    } catch (e) {
-      console.error("Exception caught: " + e.message, "\r\nURL:", url);
-    }
-    return null;
-  }
-
-  //======= HELPER FUNCTIONS =============
-
-  getAdCategoryId(categoryText) {
-    switch (categoryText) {
-      case "Stan":
-        return AD_CATEGORY.FLAT.id;
-      case "Građevinsko zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Industrijsko zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Poljoprivredno zemljiste":
-        return AD_CATEGORY.LAND.id;
-      case "Kuća":
-        return AD_CATEGORY.HOUSE.id;
-      case "Poslovni prostor":
-        return AD_CATEGORY.OFFICE.id;
-      case "Kancelarije":
-        return AD_CATEGORY.OFFICE.id;
-      case "Apartmani":
-        return AD_CATEGORY.APARTMENT.id;
-      case "Garaža":
-        return AD_CATEGORY.GARAGE.id;
-      case "Vikendica":
-        return AD_CATEGORY.COTTAGE.id;
-      default:
-        return undefined;
-    }
-  }
-
-  getAdTypeId(adTypeText) {
-    switch (adTypeText) {
-      case "PRODAJA":
-        return AD_TYPE.AD_TYPE_SALE.stringId;
-      case "NAJAM":
-        return AD_TYPE.AD_TYPE_RENT.stringId;
-      default:
-        return undefined;
-    }
-  }
-
-  async sleep(ms) {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-
-  async saveCrawledResults(results) {
-    const savers = this.savers;
-
-    // for (const saver of savers) {
-    //   await saver.save(results);
-    // }
-
-    //For now, we use only Postgres saver, so ...
-    return savers[0].save(results);
-    //so that we can use some sequelize options and information when data is inserted
-  }
-}
-
-module.exports = SaljicCrawler;

app/helpers/db/realEstate.js

@@ -2,6 +2,8 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
+const { AD_CATEGORY } = require("../../common/enums");
+
 const bulkUpsertRealEstates = async realEstateData => {
   try {
     const fieldsToUpdateIfDuplicate = [
@@ -96,12 +98,16 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     floorMin,
     floorMax,
     includeIncompleteAds,
+    includeWithoutPrice,
     balcony,
     elevator,
     newBuilding,
     accessRoadType
   } = searchRequest;
 
+  //Needed for defining which attribute should exist or not
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+
   const longitudeColumn = sequelize.col("locationLong");
   const latitudeColumn = sequelize.col("locationLat");
 
@@ -134,15 +140,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const query = {
     adType,
     realEstateType,
-    price: {
-      [Op.or]: {
-        [Op.and]: {
-          [Op.lte]: priceMax,
-          [Op.gte]: priceMin
-        },
-        [Op.is]: null
-      }
-    },
     area: {
       [Op.lte]: sizeMax,
       [Op.gte]: sizeMin
@@ -154,15 +151,6 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const queryIncludeIncomplete = {
     adType,
     realEstateType,
-    price: {
-      [Op.or]: {
-        [Op.and]: {
-          [Op.lte]: priceMax,
-          [Op.gte]: priceMin
-        },
-        [Op.is]: null
-      }
-    },
     area: {
       [Op.or]: {
         [Op.and]: {
@@ -175,8 +163,49 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     [Op.and]: geoSearchQueryPart
   };
 
-  //Every other attribute is checked separately and included in query only if it is defined
-  if (gardenSizeMax && gardenSizeMin) {
+  //Is user unchecked includeWithoutPrice FALSE then it shouldn't return null values of price
+  //If not then null values are accepted (this is DEFAULT)
+  //includeIncpompleteAds does not have effect on price query
+  if (includeWithoutPrice) {
+    query.price = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    };
+    queryIncludeIncomplete.price = {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    };
+  } else {
+    query.price = {
+      [Op.and]: {
+        [Op.lte]: priceMax,
+        [Op.gte]: priceMin
+      }
+    };
+    queryIncludeIncomplete.price = {
+      [Op.and]: {
+        [Op.lte]: priceMax,
+        [Op.gte]: priceMin
+      }
+    };
+  }
+  //Every other attribute is checked separately and included in query only if it is defined for real estate type
+
+  if (
+    realEstateTypeObject.hasGardenSize &&
+    gardenSizeMax != null &&
+    gardenSizeMin != null
+  ) {
     query.gardenSize = {
       [Op.lte]: gardenSizeMax,
       [Op.gte]: gardenSizeMin
@@ -192,7 +221,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (numberOfRoomsMin && numberOfRoomsMax) {
+  if (
+    realEstateTypeObject.hasNumberOfRoom &&
+    numberOfRoomsMin != null &&
+    numberOfRoomsMax != null
+  ) {
     query.numberOfRooms = {
       [Op.lte]: numberOfRoomsMax,
       [Op.gte]: numberOfRoomsMin
@@ -208,7 +241,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (numberOfFloorsMin && numberOfFloorsMax) {
+  if (
+    realEstateTypeObject.hasNumberOfFloors &&
+    numberOfFloorsMin != null &&
+    numberOfFloorsMax != null
+  ) {
     query.numberOfFloors = {
       [Op.lte]: numberOfFloorsMax,
       [Op.gte]: numberOfFloorsMin
@@ -224,7 +261,11 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (floorMin && floorMax) {
+  if (
+    realEstateTypeObject.hasFloorProp &&
+    floorMin != null &&
+    floorMax != null
+  ) {
     query.floor = {
       [Op.lte]: floorMax,
       [Op.gte]: floorMin
@@ -239,8 +280,10 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-
-  if (balcony) {
+  //Logic for balcony, newBuilding and elevator from users side
+  //If true is checked, then I want characteristic to be true but,
+  //if it is not checked, then I dont care - it can be null or false or true
+  if (realEstateTypeObject.hasBalconyProp && balcony === true) {
     query.balcony = {
       [Op.eq]: balcony
     };
@@ -252,7 +295,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (newBuilding) {
+  if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) {
     query.newBuilding = {
       [Op.eq]: newBuilding
     };
@@ -264,7 +307,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (elevator) {
+  if (realEstateTypeObject.hasElevatorProp && elevator === true) {
     query.elevator = {
       [Op.eq]: elevator
     };
@@ -275,7 +318,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-
+  //If user wants 'ANY' road type acces then it is not included in query -
+  //returns every road type and null values
   if (accessRoadType !== "ANY") {
     query.accessRoadType = {
       [Op.eq]: accessRoadType

app/helpers/db/searchRequest.js

@@ -49,128 +49,390 @@ const findSearchRequestsForRealEstate = async realEstate => {
 
   const geoSearchQueryPart = sequelize.where(contains, true);
 
-  //General query contains only attributes that are defined for every RealEstate - not null
-  const query = {
-    adType,
-    realEstateType,
-    subscribed: true,
-    [Op.and]: geoSearchQueryPart
-  };
   //Needed for defining which attribute should exist or not
   const realEstateTypeObject = AD_CATEGORY[realEstateType];
-  //Needed to decide on including incomplete RealEstates data
+
+  // ?? Needed to decide on including incomplete RealEstates data
   let checkForIncompleteWanted = false;
 
-  //Attributes are checked separately and included in query only if defined
-  //Price and area should be defined for every property
+  //Attributes are checked separately to make different query parts
 
-  if (price) {
-    query.priceMin = {
-      [Op.lte]: price
+  //If real estate price is number then it searches for req that have priceMin and priceMax
+  //If real estate price is null it searches for req that accept ads without price
+  //User always defines price and area (sliders) - not null in search req
+  let priceQuery = {};
+  if (price != null) {
+    priceQuery = {
+      [Op.and]: [
+        {
+          priceMin: {
+            [Op.lte]: price
+          }
+        },
+        {
+          priceMax: {
+            [Op.gte]: price
+          }
+        }
+      ]
     };
-    query.priceMax = {
-      [Op.gte]: price
+  } else {
+    priceQuery = {
+      includeWithoutPrice: {
+        [Op.eq]: true
+      }
     };
   }
 
-  if (area) {
-    query.sizeMin = {
-      [Op.lte]: area
-    };
-    query.sizeMax = {
-      [Op.gte]: area
+  let areaQuery = {};
+  if (area != null) {
+    areaQuery = {
+      [Op.and]: [
+        {
+          sizeMin: {
+            [Op.lte]: area
+          }
+        },
+        {
+          sizeMax: {
+            [Op.gte]: area
+          }
+        }
+      ]
     };
   } else {
     checkForIncompleteWanted = true;
   }
+
   //Other attributes can be defined or not depending on RealEstate type
-  if (gardenSize) {
-    query.gardenSizeMin = {
-      [Op.lte]: gardenSize
-    };
-    query.gardenSizeMax = {
-      [Op.gte]: gardenSize
-    };
-  } else if (realEstateTypeObject.hasGardenSize) {
-    checkForIncompleteWanted = true;
+  //we check what to include in query based on real estate type object
+  let gardenSizeQuery = {};
+  if (realEstateTypeObject.hasGardenSize) {
+    if (gardenSize != null) {
+      gardenSizeQuery = {
+        [Op.and]: [
+          {
+            gardenSizeMin: {
+              [Op.lte]: gardenSize
+            }
+          },
+          {
+            gardenSizeMax: {
+              [Op.gte]: gardenSize
+            }
+          }
+        ]
+      };
+    } else {
+      checkForIncompleteWanted = true;
+    }
   }
 
-  if (numberOfRooms) {
-    query.numberOfRoomsMin = {
-      [Op.lte]: numberOfRooms
-    };
-    query.numberOfRoomsMax = {
-      [Op.gte]: numberOfRooms
-    };
-  } else if (realEstateTypeObject.hasNumberOfRoom) {
-    checkForIncompleteWanted = true;
+  let numberOfRoomsQuery = {};
+  if (realEstateTypeObject.hasNumberOfRoom) {
+    if (numberOfRooms != null) {
+      //If real estate has defined number of rooms ex. 3 it returns req
+      // that accepts 3 rooms or ones that don't have defined number - null
+      //Ex. they didnt choose advanced filters at all
+      numberOfRoomsQuery = {
+        [Op.and]: [
+          {
+            numberOfRoomsMin: {
+              [Op.or]: {
+                [Op.lte]: numberOfRooms,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            numberOfRoomsMax: {
+              [Op.or]: {
+                [Op.gte]: numberOfRooms,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      // If real estate dont have defined number of rooms ex. null
+      //It returns requests that didn't choose number of rooms - also null
+      //Or ones that picked some values but also picked to includeIncomplete ads
+      numberOfRoomsQuery = {
+        [Op.or]: [
+          {
+            [Op.and]: [
+              {
+                numberOfRoomsMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                numberOfRoomsMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.eq]: true
+            }
+          }
+        ]
+      };
+    }
+  }
+  //Same logic for number of Floors and floors
+  let numberOfFloorsQuery = {};
+  if (realEstateTypeObject.hasNumberOfFloors) {
+    if (numberOfFloors != null) {
+      numberOfFloorsQuery = {
+        [Op.and]: [
+          {
+            numberOfFloorsMin: {
+              [Op.or]: {
+                [Op.lte]: numberOfFloors,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            numberOfFloorsMax: {
+              [Op.or]: {
+                [Op.gte]: numberOfFloors,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      numberOfFloorsQuery = {
+        [Op.or]: [
+          {
+            [Op.and]: [
+              {
+                numberOfFloorsMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                numberOfFloorsMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.eq]: true
+            }
+          }
+        ]
+      };
+    }
+  }
+  let floorQuery = {};
+  if (realEstateTypeObject.hasFloorProp) {
+    if (floor != null) {
+      floorQuery = {
+        [Op.and]: [
+          {
+            floorMin: {
+              [Op.or]: {
+                [Op.lte]: floor,
+                [Op.is]: null
+              }
+            }
+          },
+          {
+            floorMax: {
+              [Op.or]: {
+                [Op.gte]: floor,
+                [Op.is]: null
+              }
+            }
+          }
+        ]
+      };
+    } else {
+      floorQuery = {
+        [Op.or]: [
+          {
+            [Op.and]: [
+              {
+                floorMin: {
+                  [Op.is]: null
+                }
+              },
+              {
+                floorMax: {
+                  [Op.is]: null
+                }
+              }
+            ]
+          },
+          {
+            includeIncompleteAds: {
+              [Op.eq]: true
+            }
+          }
+        ]
+      };
+    }
   }
 
-  if (numberOfFloors) {
-    query.numberOfFloorsMin = {
-      [Op.lte]: numberOfFloors
-    };
-    query.numberOfFloorsMax = {
-      [Op.gte]: numberOfFloors
-    };
-  } else if (realEstateTypeObject.hasNumberOfFloors) {
-    checkForIncompleteWanted = true;
+  //Logic for balcony, newBuilding and elevator
+  //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
+  //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
+  //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
+  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
+  let balconyQuery = {};
+  if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
+    if (balcony === false) {
+      balconyQuery = {
+        balcony: {
+          [Op.ne]: true
+        }
+      };
+    } else if (balcony === null) {
+      balconyQuery = {
+        [Op.or]: [
+          {
+            balcony: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                balcony: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.eq]: true
+                }
+              }
+            ]
+          }
+        ]
+      };
+    }
   }
-
-  if (floor) {
-    query.floorMin = {
-      [Op.lte]: floor
-    };
-    query.floorMax = {
-      [Op.gte]: floor
-    };
-  } else if (realEstateTypeObject.hasFloorProp) {
-    checkForIncompleteWanted = true;
+  let newBuildingQuery = {};
+  if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) {
+    if (newBuilding === false) {
+      newBuildingQuery = {
+        newBuilding: {
+          [Op.ne]: true
+        }
+      };
+    } else if (newBuilding === null) {
+      newBuildingQuery = {
+        [Op.or]: [
+          {
+            newBuilding: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                newBuilding: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.eq]: true
+                }
+              }
+            ]
+          }
+        ]
+      };
+    }
   }
+  let elevatorQuery = {};
+  if (realEstateTypeObject.hasElevatorProp && elevator !== true) {
+    if (elevator === false) {
+      elevatorQuery = {
+        elevator: {
+          [Op.ne]: true
+        }
+      };
+    } else if (elevator === null) {
+      elevatorQuery = {
+        [Op.or]: [
+          {
+            elevator: {
+              [Op.ne]: true
+            }
+          },
+          {
+            [Op.and]: [
+              {
+                elevator: {
+                  [Op.eq]: true
+                }
+              },
+              {
+                includeIncompleteAds: {
+                  [Op.eq]: true
+                }
+              }
+            ]
+          }
+        ]
+      };
+    }
+  }
+  //General query consists of each individual query
+  const query = {
+    adType,
+    realEstateType,
+    subscribed: true,
+    [Op.and]: [
+      geoSearchQueryPart,
+      priceQuery,
+      areaQuery,
+      gardenSizeQuery,
+      numberOfRoomsQuery,
+      numberOfFloorsQuery,
+      floorQuery,
+      balconyQuery,
+      newBuildingQuery,
+      elevatorQuery
+    ]
+  };
 
-  if (accessRoadType) {
+  //AccessRoadType is defined - should exists for each ad and estate type
+  if (accessRoadType != null) {
     query.accessRoadType = {
       [Op.or]: {
-        [Op.eq]: "ANY",
+        [Op.like]: "ANY",
         [Op.eq]: accessRoadType
       }
     };
-  } else if (realEstateTypeObject.hasAccesRoadType) {
-    checkForIncompleteWanted = true;
-  }
-
-  if (balcony) {
-    query.balcony = {
-      [Op.eq]: balcony
+  } else {
+    //Null values are returned for user request that wanted ANY acces road type
+    query.accessRoadType = {
+      [Op.eq]: "ANY"
     };
-  } else if (realEstateTypeObject.hasBalconyProp) {
-    checkForIncompleteWanted = true;
   }
-
-  if (newBuilding) {
-    query.newBuilding = {
-      [Op.eq]: newBuilding
-    };
-  } else if (realEstateTypeObject.hasNewBuildingProp) {
-    checkForIncompleteWanted = true;
-  }
-
-  if (elevator) {
-    query.elevator = {
-      [Op.eq]: elevator
-    };
-  } else if (realEstateTypeObject.hasElevatorProp) {
-    checkForIncompleteWanted = true;
-  }
-
-  //If one of the attributes that exists for property type is null
-  //we include in query to check if incomplete real estates are accepted
+  //Tag to check if incomplete ads are accepted in query
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
       [Op.eq]: true
     };
   }
-  return await db.SearchRequest.findAll({ where: query });
+
+  return await db.SearchRequest.findAll({
+    where: query
+  });
 };
 
 module.exports = {

app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "includeWithoutPrice", {
+      type: Sequelize.BOOLEAN,
+      defaultValue: true
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "includeWithoutPrice");
+  }
+};

app/models/searchRequest.js

@@ -15,7 +15,15 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
+        coordinates: [
+          [
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0],
+            [0, 0]
+          ]
+        ],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -71,6 +79,7 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.TEXT
     },
     includeIncompleteAds: DataTypes.BOOLEAN,
+    includeWithoutPrice: DataTypes.BOOLEAN,
     balcony: DataTypes.BOOLEAN,
     elevator: DataTypes.BOOLEAN,
     newBuilding: DataTypes.BOOLEAN,

app/views/standardFilters.ejs

@@ -18,6 +18,15 @@
   </div>
 </div>
 
+<br />
+<p class="distinguished">
+  <label class="checkbox-label">
+    <input type="checkbox" class="filled-in" name="includeWithoutPrice"
+      checked
+      >
+    <span>Uključi i oglase bez cijene</span>
+  </label>
+</p>
 <br />
 
 <div class="row center-align">

development.env

@@ -59,8 +59,3 @@ AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to
 AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
-#==SALJIC NEKRETNINE==
-SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
-SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
-SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
-SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found