Added crawler for Saljic nekretnine.

.prettierignore

@@ -1 +0,0 @@
-*.ejs

app/common/enums.js

@@ -223,7 +223,8 @@ const AD_AGENCY = {
   OLX: "OLX",
   RENTAL: "RENTAL",
   PROSTOR: "PROSTOR",
-  AKTIDO: "AKTIDO"
+  AKTIDO: "AKTIDO",
+  SALJIC: "SALJIC"
 };
 
 const CRAWLER_AD_TYPE = {

app/controllers/realEstateFilters.js

@@ -35,8 +35,7 @@ const getFilters = async (req, res) => {
     balcony,
     elevator,
     newBuilding,
-    accessRoadType,
+    accessRoadType
-    includeWithoutPrice
   } = searchRequest;
   const category = AD_CATEGORY[realEstateType] || AD_CATEGORY.FLAT;
 
@@ -116,8 +115,7 @@ const getFilters = async (req, res) => {
     advancedSegmentSelectFilterValues,
     advancedRangeFilterObjects,
     advancedRangeFilterValues,
-    includeIncompleteAds,
+    includeIncompleteAds
-    includeWithoutPrice
   });
 };
 
@@ -193,7 +191,6 @@ const postFilters = async (req, res) => {
   });
 
   const includeIncompleteAds = req.body.includeIncompleteAds === "on";
-  const includeWithoutPrice = req.body.includeWithoutPrice === "on";
 
   const balcony = req.body.balcony === "on";
   const elevator = req.body.elevator === "on";
@@ -220,7 +217,6 @@ const postFilters = async (req, res) => {
   searchRequest.newBuilding = newBuilding;
 
   searchRequest.includeIncompleteAds = includeIncompleteAds;
-  searchRequest.includeWithoutPrice = includeWithoutPrice;
 
   searchRequest.accessRoadType = accessRoadType;
 

app/crawler/crawl.js

@@ -9,12 +9,14 @@ const OlxCrawler = require("./specificCrawlers/olx");
 const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
+const SaljicCrawler = require("./specificCrawlers/saljic");
 
 const {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
 } = require("./crawlerConfig");
 const PostgresSaver = require("./savers/postgres");
 
@@ -57,6 +59,15 @@ async function crawlAll() {
       AKTIDO_CONFIG.AKTIDO_MAX_RESULTS_PER_PAGE,
       AKTIDO_CONFIG.AKTIDO_IGNORED_USERNAMES,
       AKTIDO_CONFIG.AKTIDO_DELAY_BETWEEN_PAGES
+    ),
+    new SaljicCrawler(
+      [postgresSaver],
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_TYPE,
+      SALJIC_CONFIG.SALJIC_CRAWLER_AD_CATEGORIES,
+      SALJIC_CONFIG.SALJIC_MAX_PAGES,
+      SALJIC_CONFIG.SALJIC_MAX_RESULTS_PER_PAGE,
+      SALJIC_CONFIG.SALJIC_IGNORED_USERNAMES,
+      SALJIC_CONFIG.SALJIC_DELAY_BETWEEN_PAGES
     )
   ];
 

app/crawler/crawlerConfig.js

@@ -5,10 +5,12 @@ const OLX_CONFIG = require("./specificConfigs/olx");
 const RENTAL_CONFIG = require("./specificConfigs/rental");
 const PROSTOR_CONFIG = require("./specificConfigs/prostor");
 const AKTIDO_CONFIG = require("./specificConfigs/aktido");
+const SALJIC_CONFIG = require("./specificConfigs/saljic");
 
 module.exports = {
   OLX_CONFIG,
   RENTAL_CONFIG,
   PROSTOR_CONFIG,
-  AKTIDO_CONFIG
+  AKTIDO_CONFIG,
+  SALJIC_CONFIG
 };

app/crawler/specificConfigs/saljic.js

@@ -0,0 +1,34 @@
+"use strict";
+const { CRAWLER_AD_TYPE, AD_CATEGORY } = require("../../common/enums");
+
+const saljicCrawlerAdType =
+  process.env.SALJIC_CRAWLER_AD_TYPE !== undefined
+    ? CRAWLER_AD_TYPE[process.env.SALJIC_CRAWLER_AD_TYPE]
+    : null;
+
+const saljicParsedCrawlerAdCategories =
+  process.env.SALJIC_CRAWLER_AD_CATEGORIES !== undefined
+    ? process.env.SALJIC_CRAWLER_AD_CATEGORIES.split(",").map(category =>
+        category.trim()
+      )
+    : ["FLAT", "HOUSE"];
+
+const saljicIgnoredUsernames = [];
+
+const transformedSaljicCrawlerAdCategories = saljicParsedCrawlerAdCategories
+  .map(categoryName =>
+    AD_CATEGORY[categoryName] ? AD_CATEGORY[categoryName].id : undefined
+  )
+  .filter(category => !!category);
+
+module.exports = {
+  SALJIC_MAX_PAGES: parseInt(process.env.SALJIC_MAX_PAGES) || 100,
+  SALJIC_MAX_RESULTS_PER_PAGE:
+    parseInt(process.env.SALJIC_MAX_RESULTS_PER_PAGE) || 5000,
+  SALJIC_CRAWLER_AD_TYPE: saljicCrawlerAdType || CRAWLER_AD_TYPE.NONE,
+  SALJIC_CRAWLER_AD_CATEGORIES: transformedSaljicCrawlerAdCategories,
+  SALJIC_IGNORED_USERNAMES: saljicIgnoredUsernames || [],
+  SALJIC_DELAY_BETWEEN_PAGES:
+    parseInt(process.env.SALJIC_DELAY_BETWEEN_PAGES) || 1000,
+  SALJIC_FORCE_CRAWL: !!parseInt(process.env.SALJIC_FORCE_CRAWL)
+};

app/crawler/specificCrawlers/saljic.js

@@ -0,0 +1,611 @@
+"use strict";
+
+const fetch = require("node-fetch");
+const cheerio = require("cheerio");
+const moment = require("moment-timezone");
+
+const {
+  AD_TYPE,
+  AD_CATEGORY,
+  AD_AGENCY,
+  AD_STATUS,
+  CRAWLER_AD_TYPE,
+  FURNISHING_TYPE,
+  HEATING_TYPE
+} = require("../../common/enums");
+
+const {
+  PRINT_CRAWLER_DEBUG,
+  DEFAULT_TIMEZONE
+} = require("../../config/appConfig");
+const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
+
+const SALJIC_ENUMS = {
+  SALJIC_AD_TYPE: {
+    [CRAWLER_AD_TYPE.ALL]: "&input_vrsta=",
+    [CRAWLER_AD_TYPE.ONLY_SELL]: "&input_vrsta=1",
+    [CRAWLER_AD_TYPE.ONLY_RENT]: "&input_vrsta=2"
+  },
+  SALJIC_AD_CATEGORY: {
+    [AD_CATEGORY.ALL.id]: "&input_kategorija=",
+    [AD_CATEGORY.FLAT.id]: "&input_kategorija=15",
+    [AD_CATEGORY.HOUSE.id]: "&input_kategorija=9",
+    [AD_CATEGORY.LAND.id]: "&input_kategorija=5", //3 and 4 also gradjevinsko
+    [AD_CATEGORY.OFFICE.id]: "&input_kategorija=8",
+    [AD_CATEGORY.APARTMENT.id]: "&input_kategorija=1",
+    [AD_CATEGORY.GARAGE.id]: "&input_kategorija=2"
+    //[AD_CATEGORY.COTTAGE.id]: ""
+  }
+};
+
+class SaljicCrawler {
+  constructor(
+    savers = [],
+    crawlerAdTypes = CRAWLER_AD_TYPE.ALL,
+    crawlerAdCategories = [AD_CATEGORY.FLAT, AD_CATEGORY.HOUSE],
+    maxPages = 5000,
+    maxResultsPerPage = 5000,
+    ignoredUsernames = [],
+    delayBetweenPages = 1000
+  ) {
+    this.savers = savers;
+    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
+    this.crawlerAdTypes = crawlerAdTypes;
+    this.crawlerAdCategories = crawlerAdCategories;
+    this.maxResultsPerPage = maxResultsPerPage;
+    this.delayBetweenPages = delayBetweenPages;
+  }
+
+  async crawl() {
+    const crawlAdCategories = this.crawlerAdCategories;
+
+    const newRealEstates = [];
+
+    if (crawlAdCategories) {
+      const indexGenerators = [];
+      for (const adCategory of crawlAdCategories) {
+        indexGenerators.push(this.categoryIndexer(adCategory));
+      }
+      //
+      //console.log(indexGenerators);
+      //
+      let done = false;
+      while (!done) {
+        const categoryIndexerPromises = [];
+        const generatorsToRemove = [];
+        for (const indexGenerator of indexGenerators) {
+          categoryIndexerPromises.push(indexGenerator.next());
+          generatorsToRemove.push(false);
+        }
+
+        const singlePageResults = await Promise.all(categoryIndexerPromises);
+        const entries = singlePageResults.entries();
+
+        for (const [index, { value: singlePageResult }] of entries) {
+          if (singlePageResult) {
+            const saveResults = await this.saveCrawledResults(singlePageResult);
+            const { newRecords } = saveResults;
+
+            newRealEstates.push(...newRecords);
+
+            if (
+              Array.isArray(newRecords) &&
+              newRecords.length === 0 &&
+              !SALJIC_FORCE_CRAWL
+            ) {
+              generatorsToRemove[index] = true;
+            }
+          } else {
+            //Generator returned undefined, remove this generator from array
+            generatorsToRemove[index] = true;
+            // console.log("Generator ", index + 1, "has no more pages");
+          }
+        }
+
+        // console.log("Generators state : ", generatorsToRemove);
+        for (let i = generatorsToRemove.length - 1; i >= 0; i--) {
+          if (generatorsToRemove[i]) {
+            // console.log("\tRemove generator ", i + 1);
+            indexGenerators.splice(i, 1);
+          }
+        }
+        if (indexGenerators.length === 0) {
+          done = true;
+        }
+
+        await this.sleep(this.delayBetweenPages);
+      }
+    }
+    return newRealEstates;
+  }
+
+  async *categoryIndexer(adCategory) {
+    let pageToIndex = 1;
+
+    const urlAdTypePart = SALJIC_ENUMS.SALJIC_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = SALJIC_ENUMS.SALJIC_AD_CATEGORY[adCategory];
+
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPagePart = pageToIndex === 1 ? "" : (pageToIndex - 1) * 2 * 11;
+        const urlPageToCrawl = `${this.baseUrl}?order_by=${urlAdTypePart}${urlCategoryPart}&per_page=${urlPagePart}`;
+
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );
+
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
+        }
+      }
+    } else {
+      return undefined;
+    }
+  }
+
+  async indexSinglePage(url, maxResultsPerPage) {
+    if (PRINT_CRAWLER_DEBUG) {
+      console.log("[SALJIC] Index page : ", url);
+    }
+
+    try {
+      const res = await fetch(url);
+      const body = await res.text();
+      const $ = cheerio.load(body);
+      let hrefs = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const href = $(elem)
+            .find("a")
+            .first()
+            .attr("href");
+          if (href) {
+            hrefs.push(href);
+          }
+        });
+
+      let adTypesTmp = [];
+
+      $("#shop")
+        .find(".product")
+        .each((i, elem) => {
+          const adType = $(elem)
+            .find(".trakica-search-page")
+            .text()
+            .trim();
+          if (adType) {
+            adTypesTmp.push(adType);
+          }
+        });
+
+      //Converting to AD_TYPE
+      const adTypes = adTypesTmp.map(adTypeText => {
+        return this.getAdTypeId(adTypeText);
+      });
+
+      //Converting to absolute URLs
+      const hrefsAbs = hrefs.map(link => {
+        return "https://www.saljicnekretnine.ba" + link;
+      });
+
+      let actualNoOfResults =
+        hrefsAbs.length <= maxResultsPerPage
+          ? hrefsAbs.length
+          : maxResultsPerPage;
+
+      const asyncScraping = [];
+      for (let i = 0; i < actualNoOfResults; i++) {
+        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+      }
+
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
+      return filteredScrapedData;
+    } catch (e) {
+      console.error("[SALJIC] Exception caught:" + e);
+      return [];
+    }
+  }
+
+  async scrapeAd(url, adType) {
+    console.log("[SALJIC] Scraping : ", url);
+    try {
+      const adPageSource = await fetch(url);
+      const body = await adPageSource.text();
+      const $ = cheerio.load(body);
+
+      // No information for status ex. PRODAN
+      const status = AD_STATUS.STATUS_NORMAL;
+      //Extracting agency ID from url
+      const agencyObjectId = parseInt(url.substring(46, url.length));
+
+      //Extracting main properties
+      const propertySelectors = {
+        title:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
+        price:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
+        streetName:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
+
+        descriptions:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
+        latAndLong:
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
+      };
+      const title = $(propertySelectors.title)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+
+      const priceText = $(propertySelectors.price)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .replace(/ {1,}/g, " ")
+        .trim();
+      const price =
+        priceText === "CIJENA NA UPIT"
+          ? null
+          : parseFloat(
+              priceText.substring(8, priceText.length - 3).replace(",", "")
+            );
+
+      const streetName = $(propertySelectors.streetName)
+        .text()
+        .replace(/(\r\n|\n|\r)/gm, "")
+        .trim();
+
+      const descriptions = $(propertySelectors.descriptions)
+        .text()
+        .trim();
+
+      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
+      const latText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("marker=") + 7,
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
+      );
+      const longText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
+        latAndLongSrc.length
+      );
+      const locationLat = parseFloat(latText) || null;
+      const locationLong = parseFloat(longText) || null;
+
+      //====== DETAIL INFORMATION FIELDS ==========
+      let area,
+        gardenSize,
+        numberOfRooms = null,
+        numberOfFloors = null,
+        floor = null,
+        accessRoadType = null,
+        heatingType = null,
+        furnishingType = null,
+        balcony = null,
+        newBuilding = null,
+        elevator = null,
+        water = null,
+        electricity = null,
+        drainageSystem = null,
+        registeredInZkBooks = null,
+        recentlyAdapted = null,
+        parking = null,
+        garage = null,
+        gas = null,
+        antiTheftDoor = null,
+        airCondition = null,
+        phoneConnection = null,
+        cableTV = null,
+        internet = null,
+        basementAttic = null,
+        storeRoom = null,
+        videoSurveillance = null,
+        alarm = null,
+        suitableForStudents = null,
+        includingBills = null,
+        animalsAllowed = null,
+        pool = null,
+        urbanPlanPermit = null,
+        buildingPermit = null,
+        utilityConnection = null,
+        distanceToRiver = null;
+      let publishedDate = null;
+      let renewedDate = null;
+      let realEstateType;
+      let numberOfViewsAgency = null;
+
+      //Extracting data - Glavne karakteristike
+      let mainFieldIndex = 1;
+      do {
+        const mainFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.list-group-item:nth-child(${mainFieldIndex})`;
+
+        const mainField = $(mainFieldSelector)
+          .text()
+          .replace(/[\n\r\t]/gm, "")
+          .trim();
+
+        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldValue = mainField
+          .substring(mainField.indexOf("  "), mainField.length)
+          .trim();
+
+        switch (mainFieldTitle) {
+          case "Površina":
+            area = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Okućnica":
+            gardenSize = parseFloat(
+              mainFieldValue.substring(0, mainFieldValue.indexOf(" "))
+            );
+            break;
+          case "Broj soba":
+            numberOfRooms = parseInt(mainFieldValue);
+            break;
+          case "Broj spratova":
+            numberOfFloors = parseInt(mainFieldValue);
+            break;
+          case "Sprat":
+            floor = parseInt(mainFieldValue);
+            break;
+          case "Godina renoviranja":
+            recentlyAdapted = true;
+            break;
+          case "Broj parking mjesta":
+            parking = true;
+            break;
+          case "Dostupno od":
+            const day = mainFieldValue.substring(0, 2);
+            const month = mainFieldValue.substring(3, 5);
+            const year = mainFieldValue.substring(6, mainFieldValue.length);
+            publishedDate = new Date(`${month}/${day}/${year}`);
+            break;
+          default:
+            break;
+        }
+
+        if (mainFieldTitle === "") {
+          break;
+        }
+        mainFieldIndex++;
+      } while (true);
+
+      //Extracting data - Sadrzaji
+      let additionalFieldIndex = 1;
+      do {
+        const additionalFieldSelector = `div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.col-md-12.bottommargin > ul > li.border-color.col-md-5.col-md-offset-1.col-md-pull-1.list-group-item-bottom:nth-child(${additionalFieldIndex})`;
+
+        const additionalField = $(additionalFieldSelector)
+          .text()
+          .trim();
+
+        if (additionalFieldIndex === 1) {
+          //Extracting data of real estate type
+          const categoryTmp = additionalField
+            .replace(/[\n\r\t]/gm, "")
+            .substring(
+              additionalField.indexOf("Kategorija") + 10,
+              additionalField.length
+            )
+            .trim();
+          realEstateType = this.getAdCategoryId(categoryTmp);
+        } else {
+          switch (additionalField) {
+            case "Internet":
+              internet = true;
+              break;
+            case "Garaža":
+              garage = true;
+              break;
+            case "Klima":
+              airCondition = true;
+              break;
+            case "Balkon":
+              balcony = true;
+              break;
+            case "Ostava":
+              storeRoom = true;
+              break;
+            case "Podrum":
+              basementAttic = true;
+              break;
+            case "Blindirana vrata":
+              antiTheftDoor = true;
+              break;
+            case "Voda":
+              water = true;
+              break;
+            case "Kablovska":
+              cableTV = true;
+              break;
+            case "Uknjiženo":
+              registeredInZkBooks = true;
+              break;
+            case "Grijanje - centralno":
+              heatingType = HEATING_TYPE.CENTRAL_CITY.id;
+              break;
+            case "Grijanje - plin":
+              heatingType = HEATING_TYPE.GAS.id;
+              break;
+            case "Grijanje - struja":
+              heatingType = HEATING_TYPE.ELECTRICITY.id;
+              break;
+            case "Grijanje":
+              heatingType = HEATING_TYPE.OTHER.id;
+              break;
+            case "Plin":
+              gas = true;
+              break;
+            case "Namješten":
+              furnishingType = FURNISHING_TYPE.FURNISHED.id;
+              break;
+            case "Alarm":
+              alarm = true;
+              break;
+            case "Video nadzor":
+              videoSurveillance = true;
+              break;
+            case "Lift":
+              elevator = true;
+              break;
+            case "Novogradnja":
+              newBuilding = true;
+              break;
+
+            default:
+              break;
+          }
+        }
+
+        if (additionalField === "") {
+          break;
+        }
+        additionalFieldIndex++;
+      } while (true);
+
+      //If no published date it takes current date of crawling
+      if (publishedDate) {
+        renewedDate = new Date();
+      } else {
+        publishedDate = new Date();
+        renewedDate = new Date();
+      }
+
+      const data = {
+        url,
+        agencyObjectId,
+        originAgencyName: AD_AGENCY.SALJIC,
+        realEstateType,
+        adType,
+        title,
+        price,
+        area,
+        gardenSize,
+        shortDescription: descriptions.substring(0, descriptions.indexOf(".")),
+        longDescription: descriptions,
+        streetNumber: 0,
+        streetName,
+        locality: "",
+        municipality: "",
+        city: "",
+        region: "",
+        entity: "",
+        country: "",
+        locationLat,
+        locationLong,
+        adStatus: status,
+        publishedDate,
+        renewedDate,
+        numberOfRooms,
+        numberOfFloors,
+        floor,
+        accessRoadType,
+        heatingType,
+        furnishingType,
+        balcony,
+        newBuilding,
+        elevator,
+        water,
+        electricity,
+        drainageSystem,
+        registeredInZkBooks,
+        recentlyAdapted,
+        parking,
+        garage,
+        gas,
+        antiTheftDoor,
+        airCondition,
+        phoneConnection,
+        cableTV,
+        internet,
+        basementAttic,
+        storeRoom,
+        videoSurveillance,
+        alarm,
+        suitableForStudents,
+        includingBills,
+        animalsAllowed,
+        pool,
+        urbanPlanPermit,
+        buildingPermit,
+        utilityConnection,
+        distanceToRiver,
+        numberOfViewsAgency
+      };
+      console.log(data);
+      return data;
+    } catch (e) {
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
+    }
+    return null;
+  }
+
+  //======= HELPER FUNCTIONS =============
+
+  getAdCategoryId(categoryText) {
+    switch (categoryText) {
+      case "Stan":
+        return AD_CATEGORY.FLAT.id;
+      case "Građevinsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Industrijsko zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Poljoprivredno zemljiste":
+        return AD_CATEGORY.LAND.id;
+      case "Kuća":
+        return AD_CATEGORY.HOUSE.id;
+      case "Poslovni prostor":
+        return AD_CATEGORY.OFFICE.id;
+      case "Kancelarije":
+        return AD_CATEGORY.OFFICE.id;
+      case "Apartmani":
+        return AD_CATEGORY.APARTMENT.id;
+      case "Garaža":
+        return AD_CATEGORY.GARAGE.id;
+      case "Vikendica":
+        return AD_CATEGORY.COTTAGE.id;
+      default:
+        return undefined;
+    }
+  }
+
+  getAdTypeId(adTypeText) {
+    switch (adTypeText) {
+      case "PRODAJA":
+        return AD_TYPE.AD_TYPE_SALE.stringId;
+      case "NAJAM":
+        return AD_TYPE.AD_TYPE_RENT.stringId;
+      default:
+        return undefined;
+    }
+  }
+
+  async sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async saveCrawledResults(results) {
+    const savers = this.savers;
+
+    // for (const saver of savers) {
+    //   await saver.save(results);
+    // }
+
+    //For now, we use only Postgres saver, so ...
+    return savers[0].save(results);
+    //so that we can use some sequelize options and information when data is inserted
+  }
+}
+
+module.exports = SaljicCrawler;

app/helpers/db/realEstate.js

@@ -2,8 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { AD_CATEGORY } = require("../../common/enums");
-
 const bulkUpsertRealEstates = async realEstateData => {
   try {
     const fieldsToUpdateIfDuplicate = [
@@ -98,16 +96,12 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     floorMin,
     floorMax,
     includeIncompleteAds,
-    includeWithoutPrice,
     balcony,
     elevator,
     newBuilding,
     accessRoadType
   } = searchRequest;
 
-  //Needed for defining which attribute should exist or not
-  const realEstateTypeObject = AD_CATEGORY[realEstateType];
-
   const longitudeColumn = sequelize.col("locationLong");
   const latitudeColumn = sequelize.col("locationLat");
 
@@ -140,6 +134,15 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const query = {
     adType,
     realEstateType,
+    price: {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    },
     area: {
       [Op.lte]: sizeMax,
       [Op.gte]: sizeMin
@@ -151,6 +154,15 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
   const queryIncludeIncomplete = {
     adType,
     realEstateType,
+    price: {
+      [Op.or]: {
+        [Op.and]: {
+          [Op.lte]: priceMax,
+          [Op.gte]: priceMin
+        },
+        [Op.is]: null
+      }
+    },
     area: {
       [Op.or]: {
         [Op.and]: {
@@ -163,49 +175,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     [Op.and]: geoSearchQueryPart
   };
 
-  //Is user unchecked includeWithoutPrice FALSE then it shouldn't return null values of price
+  //Every other attribute is checked separately and included in query only if it is defined
-  //If not then null values are accepted (this is DEFAULT)
+  if (gardenSizeMax && gardenSizeMin) {
-  //includeIncpompleteAds does not have effect on price query
-  if (includeWithoutPrice) {
-    query.price = {
-      [Op.or]: {
-        [Op.and]: {
-          [Op.lte]: priceMax,
-          [Op.gte]: priceMin
-        },
-        [Op.is]: null
-      }
-    };
-    queryIncludeIncomplete.price = {
-      [Op.or]: {
-        [Op.and]: {
-          [Op.lte]: priceMax,
-          [Op.gte]: priceMin
-        },
-        [Op.is]: null
-      }
-    };
-  } else {
-    query.price = {
-      [Op.and]: {
-        [Op.lte]: priceMax,
-        [Op.gte]: priceMin
-      }
-    };
-    queryIncludeIncomplete.price = {
-      [Op.and]: {
-        [Op.lte]: priceMax,
-        [Op.gte]: priceMin
-      }
-    };
-  }
-  //Every other attribute is checked separately and included in query only if it is defined for real estate type
-
-  if (
-    realEstateTypeObject.hasGardenSize &&
-    gardenSizeMax != null &&
-    gardenSizeMin != null
-  ) {
     query.gardenSize = {
       [Op.lte]: gardenSizeMax,
       [Op.gte]: gardenSizeMin
@@ -221,11 +192,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
+  if (numberOfRoomsMin && numberOfRoomsMax) {
-    realEstateTypeObject.hasNumberOfRoom &&
-    numberOfRoomsMin != null &&
-    numberOfRoomsMax != null
-  ) {
     query.numberOfRooms = {
       [Op.lte]: numberOfRoomsMax,
       [Op.gte]: numberOfRoomsMin
@@ -241,11 +208,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
+  if (numberOfFloorsMin && numberOfFloorsMax) {
-    realEstateTypeObject.hasNumberOfFloors &&
-    numberOfFloorsMin != null &&
-    numberOfFloorsMax != null
-  ) {
     query.numberOfFloors = {
       [Op.lte]: numberOfFloorsMax,
       [Op.gte]: numberOfFloorsMin
@@ -261,11 +224,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (
+  if (floorMin && floorMax) {
-    realEstateTypeObject.hasFloorProp &&
-    floorMin != null &&
-    floorMax != null
-  ) {
     query.floor = {
       [Op.lte]: floorMax,
       [Op.gte]: floorMin
@@ -280,10 +239,8 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-  //Logic for balcony, newBuilding and elevator from users side
+
-  //If true is checked, then I want characteristic to be true but,
+  if (balcony) {
-  //if it is not checked, then I dont care - it can be null or false or true
-  if (realEstateTypeObject.hasBalconyProp && balcony === true) {
     query.balcony = {
       [Op.eq]: balcony
     };
@@ -295,7 +252,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (realEstateTypeObject.hasNewBuildingProp && newBuilding === true) {
+  if (newBuilding) {
     query.newBuilding = {
       [Op.eq]: newBuilding
     };
@@ -307,7 +264,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
     };
   }
 
-  if (realEstateTypeObject.hasElevatorProp && elevator === true) {
+  if (elevator) {
     query.elevator = {
       [Op.eq]: elevator
     };
@@ -318,8 +275,7 @@ const findRealEstatesForSearchRequest = async (searchRequest, maxResults) => {
       }
     };
   }
-  //If user wants 'ANY' road type acces then it is not included in query -
+
-  //returns every road type and null values
   if (accessRoadType !== "ANY") {
     query.accessRoadType = {
       [Op.eq]: accessRoadType

app/helpers/db/searchRequest.js

@@ -49,390 +49,128 @@ const findSearchRequestsForRealEstate = async realEstate => {
 
   const geoSearchQueryPart = sequelize.where(contains, true);
 
-  //Needed for defining which attribute should exist or not
+  //General query contains only attributes that are defined for every RealEstate - not null
-  const realEstateTypeObject = AD_CATEGORY[realEstateType];
-
-  // ?? Needed to decide on including incomplete RealEstates data
-  let checkForIncompleteWanted = false;
-
-  //Attributes are checked separately to make different query parts
-
-  //If real estate price is number then it searches for req that have priceMin and priceMax
-  //If real estate price is null it searches for req that accept ads without price
-  //User always defines price and area (sliders) - not null in search req
-  let priceQuery = {};
-  if (price != null) {
-    priceQuery = {
-      [Op.and]: [
-        {
-          priceMin: {
-            [Op.lte]: price
-          }
-        },
-        {
-          priceMax: {
-            [Op.gte]: price
-          }
-        }
-      ]
-    };
-  } else {
-    priceQuery = {
-      includeWithoutPrice: {
-        [Op.eq]: true
-      }
-    };
-  }
-
-  let areaQuery = {};
-  if (area != null) {
-    areaQuery = {
-      [Op.and]: [
-        {
-          sizeMin: {
-            [Op.lte]: area
-          }
-        },
-        {
-          sizeMax: {
-            [Op.gte]: area
-          }
-        }
-      ]
-    };
-  } else {
-    checkForIncompleteWanted = true;
-  }
-
-  //Other attributes can be defined or not depending on RealEstate type
-  //we check what to include in query based on real estate type object
-  let gardenSizeQuery = {};
-  if (realEstateTypeObject.hasGardenSize) {
-    if (gardenSize != null) {
-      gardenSizeQuery = {
-        [Op.and]: [
-          {
-            gardenSizeMin: {
-              [Op.lte]: gardenSize
-            }
-          },
-          {
-            gardenSizeMax: {
-              [Op.gte]: gardenSize
-            }
-          }
-        ]
-      };
-    } else {
-      checkForIncompleteWanted = true;
-    }
-  }
-
-  let numberOfRoomsQuery = {};
-  if (realEstateTypeObject.hasNumberOfRoom) {
-    if (numberOfRooms != null) {
-      //If real estate has defined number of rooms ex. 3 it returns req
-      // that accepts 3 rooms or ones that don't have defined number - null
-      //Ex. they didnt choose advanced filters at all
-      numberOfRoomsQuery = {
-        [Op.and]: [
-          {
-            numberOfRoomsMin: {
-              [Op.or]: {
-                [Op.lte]: numberOfRooms,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            numberOfRoomsMax: {
-              [Op.or]: {
-                [Op.gte]: numberOfRooms,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      // If real estate dont have defined number of rooms ex. null
-      //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads
-      numberOfRoomsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfRoomsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfRoomsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-  //Same logic for number of Floors and floors
-  let numberOfFloorsQuery = {};
-  if (realEstateTypeObject.hasNumberOfFloors) {
-    if (numberOfFloors != null) {
-      numberOfFloorsQuery = {
-        [Op.and]: [
-          {
-            numberOfFloorsMin: {
-              [Op.or]: {
-                [Op.lte]: numberOfFloors,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            numberOfFloorsMax: {
-              [Op.or]: {
-                [Op.gte]: numberOfFloors,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      numberOfFloorsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfFloorsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfFloorsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-  let floorQuery = {};
-  if (realEstateTypeObject.hasFloorProp) {
-    if (floor != null) {
-      floorQuery = {
-        [Op.and]: [
-          {
-            floorMin: {
-              [Op.or]: {
-                [Op.lte]: floor,
-                [Op.is]: null
-              }
-            }
-          },
-          {
-            floorMax: {
-              [Op.or]: {
-                [Op.gte]: floor,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
-      };
-    } else {
-      floorQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                floorMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                floorMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.eq]: true
-            }
-          }
-        ]
-      };
-    }
-  }
-
-  //Logic for balcony, newBuilding and elevator
-  //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
-  //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
-  //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
-  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
-  let balconyQuery = {};
-  if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
-    if (balcony === false) {
-      balconyQuery = {
-        balcony: {
-          [Op.ne]: true
-        }
-      };
-    } else if (balcony === null) {
-      balconyQuery = {
-        [Op.or]: [
-          {
-            balcony: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                balcony: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  let newBuildingQuery = {};
-  if (realEstateTypeObject.hasNewBuildingProp && newBuilding !== true) {
-    if (newBuilding === false) {
-      newBuildingQuery = {
-        newBuilding: {
-          [Op.ne]: true
-        }
-      };
-    } else if (newBuilding === null) {
-      newBuildingQuery = {
-        [Op.or]: [
-          {
-            newBuilding: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                newBuilding: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  let elevatorQuery = {};
-  if (realEstateTypeObject.hasElevatorProp && elevator !== true) {
-    if (elevator === false) {
-      elevatorQuery = {
-        elevator: {
-          [Op.ne]: true
-        }
-      };
-    } else if (elevator === null) {
-      elevatorQuery = {
-        [Op.or]: [
-          {
-            elevator: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                elevator: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.eq]: true
-                }
-              }
-            ]
-          }
-        ]
-      };
-    }
-  }
-  //General query consists of each individual query
   const query = {
     adType,
     realEstateType,
     subscribed: true,
-    [Op.and]: [
+    [Op.and]: geoSearchQueryPart
-      geoSearchQueryPart,
-      priceQuery,
-      areaQuery,
-      gardenSizeQuery,
-      numberOfRoomsQuery,
-      numberOfFloorsQuery,
-      floorQuery,
-      balconyQuery,
-      newBuildingQuery,
-      elevatorQuery
-    ]
   };
+  //Needed for defining which attribute should exist or not
+  const realEstateTypeObject = AD_CATEGORY[realEstateType];
+  //Needed to decide on including incomplete RealEstates data
+  let checkForIncompleteWanted = false;
 
-  //AccessRoadType is defined - should exists for each ad and estate type
+  //Attributes are checked separately and included in query only if defined
-  if (accessRoadType != null) {
+  //Price and area should be defined for every property
+
+  if (price) {
+    query.priceMin = {
+      [Op.lte]: price
+    };
+    query.priceMax = {
+      [Op.gte]: price
+    };
+  }
+
+  if (area) {
+    query.sizeMin = {
+      [Op.lte]: area
+    };
+    query.sizeMax = {
+      [Op.gte]: area
+    };
+  } else {
+    checkForIncompleteWanted = true;
+  }
+  //Other attributes can be defined or not depending on RealEstate type
+  if (gardenSize) {
+    query.gardenSizeMin = {
+      [Op.lte]: gardenSize
+    };
+    query.gardenSizeMax = {
+      [Op.gte]: gardenSize
+    };
+  } else if (realEstateTypeObject.hasGardenSize) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (numberOfRooms) {
+    query.numberOfRoomsMin = {
+      [Op.lte]: numberOfRooms
+    };
+    query.numberOfRoomsMax = {
+      [Op.gte]: numberOfRooms
+    };
+  } else if (realEstateTypeObject.hasNumberOfRoom) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (numberOfFloors) {
+    query.numberOfFloorsMin = {
+      [Op.lte]: numberOfFloors
+    };
+    query.numberOfFloorsMax = {
+      [Op.gte]: numberOfFloors
+    };
+  } else if (realEstateTypeObject.hasNumberOfFloors) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (floor) {
+    query.floorMin = {
+      [Op.lte]: floor
+    };
+    query.floorMax = {
+      [Op.gte]: floor
+    };
+  } else if (realEstateTypeObject.hasFloorProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (accessRoadType) {
     query.accessRoadType = {
       [Op.or]: {
-        [Op.like]: "ANY",
+        [Op.eq]: "ANY",
         [Op.eq]: accessRoadType
       }
     };
-  } else {
+  } else if (realEstateTypeObject.hasAccesRoadType) {
-    //Null values are returned for user request that wanted ANY acces road type
+    checkForIncompleteWanted = true;
-    query.accessRoadType = {
-      [Op.eq]: "ANY"
-    };
   }
-  //Tag to check if incomplete ads are accepted in query
+
+  if (balcony) {
+    query.balcony = {
+      [Op.eq]: balcony
+    };
+  } else if (realEstateTypeObject.hasBalconyProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (newBuilding) {
+    query.newBuilding = {
+      [Op.eq]: newBuilding
+    };
+  } else if (realEstateTypeObject.hasNewBuildingProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  if (elevator) {
+    query.elevator = {
+      [Op.eq]: elevator
+    };
+  } else if (realEstateTypeObject.hasElevatorProp) {
+    checkForIncompleteWanted = true;
+  }
+
+  //If one of the attributes that exists for property type is null
+  //we include in query to check if incomplete real estates are accepted
   if (checkForIncompleteWanted) {
     query.includeIncompleteAds = {
       [Op.eq]: true
     };
   }
-
+  return await db.SearchRequest.findAll({ where: query });
-  return await db.SearchRequest.findAll({
-    where: query
-  });
 };
 
 module.exports = {

app/migrations/20200123085754-add-column-includeWithoutPrice-to-searchRequests-table.js

@@ -1,14 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn("SearchRequests", "includeWithoutPrice", {
-      type: Sequelize.BOOLEAN,
-      defaultValue: true
-    });
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn("SearchRequests", "includeWithoutPrice");
-  }
-};

app/models/searchRequest.js

@@ -15,15 +15,7 @@ module.exports = (sequelize, DataTypes) => {
       allowNull: false,
       defaultValue: {
         type: "Polygon",
-        coordinates: [
+        coordinates: [[[0, 0], [0, 0], [0, 0], [0, 0], [0, 0]]],
-          [
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0],
-            [0, 0]
-          ]
-        ],
         crs: { type: "name", properties: { name: "EPSG:4326" } }
       }
     },
@@ -79,7 +71,6 @@ module.exports = (sequelize, DataTypes) => {
       type: DataTypes.TEXT
     },
     includeIncompleteAds: DataTypes.BOOLEAN,
-    includeWithoutPrice: DataTypes.BOOLEAN,
     balcony: DataTypes.BOOLEAN,
     elevator: DataTypes.BOOLEAN,
     newBuilding: DataTypes.BOOLEAN,

app/views/standardFilters.ejs

@@ -18,15 +18,6 @@
   </div>
 </div>
 
-<br />
-<p class="distinguished">
-  <label class="checkbox-label">
-    <input type="checkbox" class="filled-in" name="includeWithoutPrice"
-      checked
-      >
-    <span>Uključi i oglase bez cijene</span>
-  </label>
-</p>
 <br />
 
 <div class="row center-align">

development.env

@@ -59,3 +59,8 @@ AKTIDO_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to
 AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+#==SALJIC NEKRETNINE==
+SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
+SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
+SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
+SALJIC_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found