Merge branch 'master' after user-agent change into email-density

New query for search req search.
Changed queries logic again.
2020-02-21 14:26:33 +01:00 · 2020-02-21 14:25:10 +01:00 · 2020-02-18 15:04:26 +01:00 · 2020-02-17 23:24:55 +01:00
23 changed files with 175 additions and 623 deletions
--- a/app/config/appConfig.js
+++ b/app/config/appConfig.js
@@ -45,11 +45,6 @@ const USER_AGENT =
  process.env.USER_AGENT ||
  "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";

-const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
-const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
-const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
-const NODE_FETCH_TIMEOUT_MS = parseInt(process.env.NODE_FETCH_TIMEOUT_MS) || 60000
-
 module.exports = {
  APP_PORT,
  APP_URL,
@@ -64,9 +59,5 @@ module.exports = {
  STAGING,
  CHECK_UP_DAYS,
  PROSTOR_LOGIN,
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY,
-  SCRAPER_API_BASE_URL,
-  NODE_FETCH_TIMEOUT_MS
+  USER_AGENT
 };
--- a/app/crawler/crawl.js
+++ b/app/crawler/crawl.js
@@ -10,7 +10,6 @@ const RentalCrawler = require("./specificCrawlers/rental");
 const ProstorCrawler = require("./specificCrawlers/prostor");
 const AktidoCrawler = require("./specificCrawlers/aktido");
 const SaljicCrawler = require("./specificCrawlers/saljic");
-const { logDebug } = require("../helpers/log");

 const {
  OLX_CONFIG,
@@ -76,9 +75,7 @@ async function crawlAll() {

  for (const crawler of crawlers) {
    try {
-      logDebug('Starting crawler: ', crawler);
      const newRealEstatesFromSingleCrawler = await crawler.crawl();
-      logDebug('Crawler done: ', crawler);
      if (Array.isArray(newRealEstatesFromSingleCrawler)) {
        newRealEstates.push(...newRealEstatesFromSingleCrawler);
      }
--- a/app/crawler/specificCrawlers/aktido.js
+++ b/app/crawler/specificCrawlers/aktido.js
@@ -159,7 +159,7 @@ class AktidoCrawler {
    }

    try {
-      const res = await fetch(url, {}, false);
+      const res = await fetch(url);
      const body = await res.text();
      const $ = cheerio.load(body);
      let hrefs = [];
@@ -202,10 +202,6 @@ class AktidoCrawler {
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
      const mapElementParent = $(".box-map").parent();
      const scriptElement = $("script", mapElementParent);
      if (
--- a/app/crawler/specificCrawlers/olx.js
+++ b/app/crawler/specificCrawlers/olx.js
@@ -1,7 +1,6 @@
 "use strict";

 const fetch = require("../../helpers/fetchWrapper");
-const { logDebug } = require("../../helpers/log");
 const cheerio = require("cheerio");
 const Promise = require("bluebird");
 const moment = require("moment-timezone");
@@ -45,16 +44,6 @@ const OLX_ENUMS = {

 const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");

-const chunk = (array, size = 10) => {
-  let i, j ,temparray;
-  const result = []
-  for (i=0,j=array.length; i<j; i+=size) {
-    temparray = array.slice(i,i+size);
-    result.push(temparray);
-  }
-  return result;
-}
-
 class OlxCrawler {
  constructor(
    savers = [],
@@ -63,7 +52,7 @@ class OlxCrawler {
    maxPages = 1000,
    maxResultsPerPage = 100,
    ignoredUsernames = [],
-    delayBetweenPages = 500
+    delayBetweenPages = 1000
  ) {
    this.savers = savers;
    this.baseUrl = "https://www.olx.ba/pretraga?sort_order=desc&sort_po=datum";
@@ -76,7 +65,6 @@ class OlxCrawler {
  }

  async crawl() {
-    logDebug("Starting OLX crawl");
    const crawlAdCategories = this.crawlerAdCategories;

    const newRealEstates = [];
@@ -100,32 +88,14 @@ class OlxCrawler {
        const entries = singlePageResults.entries();

        for (const [index, { value: singlePageResult }] of entries) {
-          if (PRINT_CRAWLER_DEBUG) {
-            console.log("================================");
-            console.log("Category Indexer index : ", index);
-          }
-
          if (singlePageResult) {
-            console.log("\tTotal entries          : ", singlePageResult.length)
            const saveResults = await this.saveCrawledResults(singlePageResult);
            const { newRecords, existingRecords } = saveResults;

-            if (PRINT_CRAWLER_DEBUG) {
-              console.log("--------------------------");
-              console.log("\tNew record URLs [", newRecords.length, "] :");
-
-              for(const newRecord of newRecords) {
-                console.log("\t\t",newRecord.url);
-              }
-
-              console.log("\t-------------------------");
-              console.log("\tExisting record URLs [", existingRecords.length, "] :");
-            }
-
            newRealEstates.push(...newRecords);

            for (const existingRecord of existingRecords) {
-              const { publishedDate, renewedDate, url } = existingRecord;
+              const { publishedDate, renewedDate } = existingRecord;

              const publishedDateMoment = moment.utc(publishedDate);
              const renewedDateMoment = moment.utc(renewedDate);
@@ -135,25 +105,13 @@ class OlxCrawler {
                "minute"
              );

-              if (PRINT_CRAWLER_DEBUG) {
-                console.log("\t\t", url);
-                console.log("\t\t\tPublished date         : ", publishedDate);
-                console.log("\t\t\tRenewed date           : ", renewedDate);
-                console.log("\t\t\tIs same (up to minute) : ", stopCrawlingThisCategory);
-              }
-
              if (stopCrawlingThisCategory && !OLX_FORCE_CRAWL) {
                generatorsToRemove[index] = true;
-                if (PRINT_CRAWLER_DEBUG) {
-                  console.log("\t\t\tStopping this category indexer");
-                }
+                // console.log("\tGenerator ", index + 1, "has no more new ads");
                break;
              }
            }
          } else {
-            if (PRINT_CRAWLER_DEBUG) {
-              console.log("\tNo more entries in this category, stopping!");
-            }
            //Generator returned undefined, remove this generator from array
            generatorsToRemove[index] = true;
            // console.log("Generator ", index + 1, "has no more pages");
@@ -178,36 +136,31 @@ class OlxCrawler {
  }

  async *categoryIndexer(adCategory) {
-    try {
-      let pageToIndex = 1;
+    let pageToIndex = 1;

-      const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
-      const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
-      if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
-        while (true) {
-          const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
-          const singlePageResults = await this.indexSinglePage(
-            urlPageToCrawl,
-            this.maxResultsPerPage
-          );
+    const urlAdTypePart = OLX_ENUMS.OLX_AD_TYPE[this.crawlerAdTypes];
+    const urlCategoryPart = OLX_ENUMS.OLX_AD_CATEGORY[adCategory];
+    if (urlAdTypePart !== undefined && urlCategoryPart !== undefined) {
+      while (true) {
+        const urlPageToCrawl = `${this.baseUrl}${urlAdTypePart}${urlCategoryPart}&stranica=${pageToIndex}`;
+        const singlePageResults = await this.indexSinglePage(
+          urlPageToCrawl,
+          this.maxResultsPerPage
+        );

-          await this.sleep(this.delayBetweenPages);
-          if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
-            yield singlePageResults;
-          } else {
-            return undefined;
-          }
-
-          ++pageToIndex;
-          if (pageToIndex === this.maxPages) {
-            return undefined;
-          }
+        if (Array.isArray(singlePageResults) && singlePageResults.length > 0) {
+          yield singlePageResults;
+        } else {
+          return undefined;
+        }
+
+        ++pageToIndex;
+        if (pageToIndex === this.maxPages) {
+          return undefined;
        }
-      } else {
-        return undefined;
      }
-    } catch (e) {
-      console.log('Error inside generator: ', e);
+    } else {
+      return undefined;
    }
  }

@@ -217,10 +170,8 @@ class OlxCrawler {
    }

    try {
-      const res = await fetch(url, {}, false);
-      logDebug("Got category results for: ", url);
+      const res = await fetch(url);
      const body = await res.text();
-      logDebug("Got category results text for: ", url);
      const $ = cheerio.load(body);
      let hrefs = [];

@@ -241,44 +192,26 @@ class OlxCrawler {

      const asyncScraping = [];
      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(hrefs[i]);
+        asyncScraping.push(this.scrapeAd(hrefs[i]));
      }

-      const allChunks = chunk(asyncScraping, 2);
-      const dataResults = []
-      for (let i = 0; i < allChunks.length; i++) {
-        const singleChunk = allChunks[i];
-        const promises = singleChunk.map(c => this.scrapeAd(c))
-        const chunkResults = await Promise.all(promises);
-        await this.sleep(this.delayBetweenPages);
-        dataResults.push(...chunkResults);
-        logDebug("Chunk results len:", chunkResults.length);
-      }
-
-
-      const filteredScrapedData = dataResults.filter(adData => !!adData);
-      logDebug("Filtered scraped data length: ", filteredScrapedData.length);
-
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
      return filteredScrapedData;
    } catch (e) {
-      console.error("Exception caught, index single page: " + e);
+      console.error("Exception caught:" + e);
      return [];
    }
  }

  async scrapeAd(url) {
-    logDebug("Scraping : ", url);
+    // console.log("Scraping : ", url);
    try {
      const adPageSource = await fetch(url);
      const body = await adPageSource.text();
      const $ = cheerio.load(body);
      let status = AD_STATUS.STATUS_NORMAL;

-      if (body.indexOf('<html') === -1) {
-        console.error("This is the body: ", body);
-        throw { message: 'Failed to fetch page !' }
-      }
-
      const propertySelectors = {
        username:
          "#lg > div.desno2.profil > div:nth-child(2) > div.vrsta1.vrsta_desno > a > div.username > span",
@@ -305,25 +238,37 @@ class OlxCrawler {

      //====== PRICE DETECTION AND EXTRACTION =====
      let price = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      const urgentPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
+      )
+        .text()
+        .trim();

-      const priceHeader = $("#pc > p.n").text().trim();
-      const priceValue = $("#pc > p:nth-child(2)").text().trim();
-      price = priceValue;
-
-      if (priceHeader.indexOf('Hitn') !== -1) {
-        // Urgent price
-        status = AD_STATUS.STATUS_URGENT;
-      }
-
-      const discountPriceTag = $("#artikal_glavni_div > div.artikal_lijevo > p:nth-child(4)").text().trim();
-      if (discountPriceTag.indexOf('Akcij') !== -1) {
-        status = AD_STATUS.STATUS_DISCOUNTED;
-        const discountPriceValues = $("#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p").text().trim();
-        // discountPriceValues contain string like "10.000 KM 7.500 KM"
-        // First price is regular, second is currently active (discounted) price
-        const bothPrices = discountPriceValues.split('KM');
-        // Now, currently active price is second element of bothPrices array
-        price = bothPrices[1] ? bothPrices[1].trim() : null;
+      if (normalPriceValue && normalPriceValue.length > 0) {
+        price = normalPriceValue;
+        if (
+          $("#pc > p.n")
+            .text()
+            .indexOf("Hitna") !== -1
+        ) {
+          status = AD_STATUS.STATUS_URGENT;
+        } else {
+          status = AD_STATUS.STATUS_NORMAL;
+        }
+      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.split("KM");
+        //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
+        if (priceValues.length > 1) {
+          price = priceValues[1].trim();
+          status = AD_STATUS.STATUS_DISCOUNTED;
+        } else {
+          throw { message: "Can't find urgent price" };
+        }
+      } else {
+        throw {
+          message: "Can't find price (it is not normal nor urgent price ?)"
+        };
      }

      //====== OTHER AD INFORMATION ===============
@@ -333,7 +278,7 @@ class OlxCrawler {

      let otherInformationDivId;
      //We need to locate DIV ID where other information are stored
-      for (let possibleId = 1; possibleId <= 30; possibleId++) {
+      for (let possibleId = 10; possibleId <= 20; possibleId++) {
        const adTypeFieldTitle = $(
          `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
        )
@@ -705,12 +650,10 @@ class OlxCrawler {
        distanceToRiver,
        numberOfViewsAgency
      };
-      //
-      //console.log("Scraped data:", data);

      return data;
    } catch (e) {
-      console.error("Exception caught scrapeAd : " + e.message, "\r\nURL:", url);
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
    }
    return null;
  }
@@ -825,9 +768,6 @@ class OlxCrawler {
    if (!priceText) {
      return NaN;
    }
-    if (priceText === "Po dogovoru") {
-      return null;
-    }
    const formattedPriceText = priceText.replace(".", "").replace(",", ".");
    return parseFloat(formattedPriceText);
  }
--- a/app/crawler/specificCrawlers/prostor.js
+++ b/app/crawler/specificCrawlers/prostor.js
@@ -4,7 +4,6 @@ const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
-const nodeFetch = require("node-fetch");

 const {
  AD_TYPE,
@@ -63,19 +62,13 @@ class ProstorCrawler {

  async crawl() {
    const crawlAdCategories = this.crawlerAdCategories;
-    const crawlAdTypes = this.crawlerAdTypes;
-    if (!crawlAdCategories || !crawlAdTypes) {
-      return []
-    }
-
-    const newRealEstates = [];
    //We need session cookie to use login privileges
    const prostorCookie = await this.getCookies();
-    //New tag to check if crawler logged in
+    //New tag to check if crawler loged in
    const login = await this.loginForScraping(PROSTOR_LOGIN, prostorCookie);
-
+    const newRealEstates = [];
    //Crawl only if login was successful
-    if (login) {
+    if (crawlAdCategories && login) {
      const indexGenerators = [];
      for (const adCategory of crawlAdCategories) {
        indexGenerators.push(this.categoryIndexer(adCategory, prostorCookie));
@@ -141,11 +134,6 @@ class ProstorCrawler {
        prostorCookie
      );

-      if (!Array.isArray(listOfAllRealEstates)){
-        console.log('[PROSTOR] Could not find real estate JSON data, check selector !');
-        return undefined;
-      }
-
      let elementToStartIndexFrom = 0;
      while (true) {
        const realEstatesForSinglePage = listOfAllRealEstates.slice(
@@ -209,16 +197,12 @@ class ProstorCrawler {

    // console.log("[PROSTOR] Scraping : ", url);
    try {
-      const adPageSource = await nodeFetch(url, {
+      const adPageSource = await fetch(url, {
        headers: { Cookie: prostorCookie }
      });
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
      // link contains part of the URL in the format of : /prodaja/stan/stup/9556
      // general form is : /actionType/realEstateType/location/realEstateID
      // linkParts contains : ['', 'actionType', 'realEstateType', 'location', 'realEstateID']
@@ -443,14 +427,14 @@ class ProstorCrawler {
    }

    try {
-      const res = await nodeFetch(url, {
+      const res = await fetch(url, {
        headers: { Cookie: prostorCookie }
      });
      const body = await res.text();
      const $ = cheerio.load(body);

      const scriptElement = $(
-          "body > div.content > div.container-fluid > script:nth-child(6)"
+        "body > div > div.container-fluid > script:nth-child(7)"
      );

      if (
@@ -607,7 +591,7 @@ class ProstorCrawler {
    formData.append("email", PROSTOR_LOGIN.EMAIL);
    formData.append("password", PROSTOR_LOGIN.PASSWORD);

-    return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
+    return fetch("https://prostor.ba/moj-prostor/prijava", {
      method: "POST",
      body: formData,
      headers: { Cookie: prostorCookie }
@@ -634,12 +618,9 @@ class ProstorCrawler {
      });
  }
  async getCookies() {
-    const getResponse = await nodeFetch(
-      "https://prostor.ba/moj-prostor/prijava",
-      {
-        headers: { Cookie: "" }
-      }
-    );
+    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
+      headers: { Cookie: "" }
+    });
    const raw = getResponse.headers.raw()["set-cookie"];
    const cookie = raw
      .map(datastring => {
--- a/app/crawler/specificCrawlers/rental.js
+++ b/app/crawler/specificCrawlers/rental.js
@@ -159,7 +159,7 @@ class RentalCrawler {
    }

    try {
-      const res = await fetch(url, {} , false);
+      const res = await fetch(url);
      const body = await res.text();
      const $ = cheerio.load(body);
      let hrefs = [];
@@ -202,10 +202,6 @@ class RentalCrawler {
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
      const mapElementParent = $(".box-map").parent();
      const scriptElement = $("script", mapElementParent);
      if (
--- a/app/crawler/specificCrawlers/saljic.js
+++ b/app/crawler/specificCrawlers/saljic.js
@@ -1,10 +1,8 @@
 "use strict";

 const fetch = require("../../helpers/fetchWrapper");
-const { getUrlParams } = require("../../helpers/url");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
-const PromisePool = require('@supercharge/promise-pool');

 const {
  AD_TYPE,
@@ -48,13 +46,12 @@ class SaljicCrawler {
    maxPages = 5000,
    maxResultsPerPage = 5000,
    ignoredUsernames = [],
-    delayBetweenPages = 500
+    delayBetweenPages = 1000
  ) {
    this.savers = savers;
    this.baseUrl = "https://www.saljicnekretnine.ba/v2/nekretnine_search";
    this.crawlerAdTypes = crawlerAdTypes;
    this.crawlerAdCategories = crawlerAdCategories;
-    this.maxPages = maxPages
    this.maxResultsPerPage = maxResultsPerPage;
    this.delayBetweenPages = delayBetweenPages;
  }
@@ -160,7 +157,7 @@ class SaljicCrawler {
    }

    try {
-      const res = await fetch(url, {}, false);
+      const res = await fetch(url);
      const body = await res.text();
      const $ = cheerio.load(body);
      let hrefs = [];
@@ -208,23 +205,11 @@ class SaljicCrawler {

      const asyncScraping = [];
      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push([hrefsAbs[i], adTypes[i]]);
+        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
      }

-      
-
-      const dataResults = []
-      const { scrapedData, errors } = await PromisePool
-        .withConcurrency(2)
-        .for(asyncScraping)
-        .process(async data => {
-          const result = await this.scrapeAd(...data)
-          await this.sleep(this.delayBetweenPages);
-          dataResults.push(result)
-          return result; //TODO: this does not work, scrapedData is null, dataResults works
-        })
-
-      const filteredScrapedData = dataResults.filter(adData => !!adData);
+      const scrapedData = await Promise.all(asyncScraping);
+      const filteredScrapedData = scrapedData.filter(adData => !!adData);
      return filteredScrapedData;
    } catch (e) {
      console.error("[SALJIC] Exception caught:" + e);
@@ -239,20 +224,10 @@ class SaljicCrawler {
      const body = await adPageSource.text();
      const $ = cheerio.load(body);

-      if (body.indexOf('<html') === -1) {
-        throw { message: 'Failed to fetch page !' }
-      }
-
      // No information for status ex. PRODAN
      const status = AD_STATUS.STATUS_NORMAL;
      //Extracting agency ID from url
-      const agencyObjectId = url
-        ? parseInt(url.substring(46, url.length))
-        : null;
-
-      if (!agencyObjectId) {
-        throw { message : 'No agency object ID - URL changed?'}
-      }
+      const agencyObjectId = parseInt(url.substring(46, url.length));

      //Extracting main properties
      const propertySelectors = {
@@ -262,10 +237,11 @@ class SaljicCrawler {
          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
        streetName:
          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > p",
+
        descriptions:
          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.toggle.toggle-bg > div.togglec >p:nth-child(1)",
        latAndLong:
-          "iframe"
+          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
      };
      const title = $(propertySelectors.title)
        .text()
@@ -296,26 +272,14 @@ class SaljicCrawler {
        .trim();

      const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      let latText;
-      let longText;
-      if (latAndLongSrc){
-        const mapParams = getUrlParams(latAndLongSrc);
-        if (mapParams) {
-          if (mapParams['marker']){
-            const marker = mapParams['marker'].split(',');
-            latText = marker[0] ? marker[0] : undefined;
-            longText = marker[1] ? marker[1] : undefined;
-          }else{
-            if (mapParams['mlat']) {
-              latText = mapParams['mlat'];
-            }
-            if (mapParams['mlon']) {
-              longText = mapParams['mlon'];
-            }
-          }
-        }
-      }
-
+      const latText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("marker=") + 7,
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
+      );
+      const longText = latAndLongSrc.substring(
+        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
+        latAndLongSrc.length
+      );
      const locationLat = parseFloat(latText) || null;
      const locationLong = parseFloat(longText) || null;

@@ -364,10 +328,11 @@ class SaljicCrawler {
      let numberOfViewsKivi = null;
      let streetNumber = 0;
      let adStatus = status;
-      let shortDescription = descriptions
-        ? descriptions.substring(0, descriptions.indexOf("."))
-        : "";
-      let longDescription = descriptions || "";
+      let shortDescription = descriptions.substring(
+        0,
+        descriptions.indexOf(".")
+      );
+      let longDescription = descriptions;
      //Extracting data - Glavne karakteristike
      let mainFieldIndex = 1;
      do {
@@ -378,14 +343,10 @@ class SaljicCrawler {
          .replace(/[\n\r\t]/gm, "")
          .trim();

-        const mainFieldTitle = mainField
-          ? mainField.substring(0, mainField.indexOf("  "))
-          : "";
+        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
        const mainFieldValue = mainField
-          ? mainField
-              .substring(mainField.indexOf("  "), mainField.length)
-              .trim()
-          : "";
+          .substring(mainField.indexOf("  "), mainField.length)
+          .trim();

        switch (mainFieldTitle) {
          case "Površina":
@@ -402,7 +363,7 @@ class SaljicCrawler {
            numberOfRooms = parseInt(mainFieldValue);
            break;
          case "Broj spratova":
-            numberOfFloors = this.parseNumberOfFloors(mainFieldValue);
+            numberOfFloors = parseInt(mainFieldValue);
            break;
          case "Sprat":
            floor = parseInt(mainFieldValue);
@@ -448,9 +409,6 @@ class SaljicCrawler {
            )
            .trim();
          realEstateType = this.getAdCategoryId(categoryTmp);
-          if (!realEstateType) {
-            throw { message: 'No real estate type - page body not loaded correctly or page changed?' }
-          }
        } else {
          switch (additionalField) {
            case "Internet":
@@ -607,7 +565,7 @@ class SaljicCrawler {

      return data;
    } catch (e) {
-      console.error("[SALJIC] Exception caught: " + e.message, "\r\nURL:", url);
+      console.error("Exception caught: " + e.message, "\r\nURL:", url);
    }
    return null;
  }
@@ -652,21 +610,6 @@ class SaljicCrawler {
    }
  }

-  parseNumberOfFloors(numberOfFloorsText) {
-    const tryNumericalValue = parseInt(numberOfFloorsText);
-    if (!isNaN(tryNumericalValue)){
-      return tryNumericalValue;
-    }
-
-    // Guess number of floors based on number of + sign concatenations
-    // e.g. P+S+Pt -> 3 floors
-    if (typeof numberOfFloorsText === 'string' && numberOfFloorsText.indexOf('+') > 0) {
-      return numberOfFloorsText.split('+').length + 1
-    }
-
-    return null
-  }
-
  async sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }
--- a/app/helpers/db/searchRequest.js
+++ b/app/helpers/db/searchRequest.js
@@ -3,7 +3,6 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
-const { CHECK_UP_DAYS } = require("../../config/appConfig");

 const getSearchRequest = async searchRequestId => {
  try {
@@ -17,22 +16,6 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
  return await db.SearchRequest.create(searchRequestFields);
 };
-const findAllRequestsForCheckUp = async () => {
-  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
-  const checkupDate = new Date();
-  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
-
-  const dateQuery = {
-    notifiedAt: {
-      [Op.lte]: checkupDate
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: dateQuery
-  });
-
-  return allRequestsForCheckUp;
-};

 const findSearchRequestsForRealEstate = async realEstate => {
  const {
@@ -173,33 +156,11 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else {
      // If real estate dont have defined number of rooms ex. null
-      //It returns requests that didn't choose number of rooms - also null
-      //Or ones that picked some values but also picked to includeIncomplete ads (or default)
+      //It returns all search requests except for ones that dont want incpomlete ads
      numberOfRoomsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfRoomsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfRoomsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
@@ -229,30 +190,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else {
      numberOfFloorsQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                numberOfFloorsMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                numberOfFloorsMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
@@ -281,30 +221,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else {
      floorQuery = {
-        [Op.or]: [
-          {
-            [Op.and]: [
-              {
-                floorMin: {
-                  [Op.is]: null
-                }
-              },
-              {
-                floorMax: {
-                  [Op.is]: null
-                }
-              }
-            ]
-          },
-          {
-            includeIncompleteAds: {
-              [Op.or]: {
-                [Op.eq]: true,
-                [Op.is]: null
-              }
-            }
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
@@ -313,7 +232,7 @@ const findSearchRequestsForRealEstate = async realEstate => {
  //If user dont check checkbox for ex. elevator  it does not mean he only wants no elevator
  //If real estate characteristic =true find all req, one that wants charachertistic or dont care - dont need query
  //If real estate characteristic = false, find all req exept for ones that wants characteristic to be true
-  //If real estate characteristic = null, dont know if true or false, find req that dont care or want char and want incomplete ads
+  //If real estate characteristic = null, dont know if true or false, find all req except ones that dont want incomplete ads
  let balconyQuery = {};
  if (realEstateTypeObject.hasBalconyProp && balcony !== true) {
    if (balcony === false) {
@@ -324,30 +243,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else if (balcony === null) {
      balconyQuery = {
-        [Op.or]: [
-          {
-            balcony: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                balcony: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
@@ -361,30 +259,9 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else if (newBuilding === null) {
      newBuildingQuery = {
-        [Op.or]: [
-          {
-            newBuilding: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                newBuilding: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
@@ -398,33 +275,13 @@ const findSearchRequestsForRealEstate = async realEstate => {
      };
    } else if (elevator === null) {
      elevatorQuery = {
-        [Op.or]: [
-          {
-            elevator: {
-              [Op.ne]: true
-            }
-          },
-          {
-            [Op.and]: [
-              {
-                elevator: {
-                  [Op.eq]: true
-                }
-              },
-              {
-                includeIncompleteAds: {
-                  [Op.or]: {
-                    [Op.eq]: true,
-                    [Op.is]: null
-                  }
-                }
-              }
-            ]
-          }
-        ]
+        includeIncompleteAds: {
+          [Op.ne]: false
+        }
      };
    }
  }
+
  //General query consists of each individual query
  const query = {
    adType,
@@ -476,6 +333,5 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
  getSearchRequest,
  createSearchRequest,
-  findSearchRequestsForRealEstate,
-  findAllRequestsForCheckUp
+  findSearchRequestsForRealEstate
 };
--- a/app/helpers/db/searchRequestMatch.js
+++ b/app/helpers/db/searchRequestMatch.js
@@ -2,6 +2,7 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
+const { CHECK_UP_DAYS } = require("../../config/appConfig");

 const findRealEstatesForSearchRequest = async searchRequestId => {
  const query = {
@@ -42,6 +43,42 @@ const findNotNotifiedMatches = async () => {

  return matchingRecords;
 };
+const findAllRequestsForCheckUp = async () => {
+  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
+  //The ones that received notification for real estate CHECK_UP_DAYS days from now
+  const date = new Date();
+  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
+  date.setDate(checkUpDate);
+  const dateQuery = {
+    createdAt: {
+      [Op.gte]: date
+    }
+  };
+
+  const excludedMatches = await db.SearchRequestMatch.findAll({
+    attributes: ["searchRequestId"],
+    where: dateQuery,
+    order: [["searchRequestId", "ASC"]]
+  });
+
+  const excludedRequestsAll = excludedMatches.map(match => {
+    return match.dataValues.searchRequestId;
+  });
+  //Removing duplicate search request id-s for optimization
+  const excludedRequests = [...new Set(excludedRequestsAll)];
+
+  const query = {
+    subscribed: true,
+    id: {
+      [Op.notIn]: excludedRequests
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: query
+  });
+
+  return allRequestsForCheckUp;
+};

 const addMatches = async matchingRecords => {
  return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -52,5 +89,6 @@ const addMatches = async matchingRecords => {
 module.exports = {
  findRealEstatesForSearchRequest,
  addMatches,
-  findNotNotifiedMatches
+  findNotNotifiedMatches,
+  findAllRequestsForCheckUp
 };
--- a/app/helpers/emailContentGenerator.js
+++ b/app/helpers/emailContentGenerator.js
@@ -9,7 +9,6 @@ const { AD_CATEGORY, AD_TYPE, EMAIL_FREQUENCY } = require("../common/enums");

 //Tag to recognize staging from development
 const stagingTag = STAGING ? "[STAGING] " : "";
-const wordOfMouthRequest = `Molimo vas <strong>recite svojim prijateljima</strong> za Kivi - što više korisnika budemo imali, moći ćemo više agencija uključiti i više nekretnina imati u bazi. Hvala!`

 const generateEmailFooter = (searchRequestId, emailFrequencyTitle) => {
  return `  <div>Trenutno ste prijavljeni da obavještenja o novim nekretninama primate <strong>${emailFrequencyTitle.toLowerCase()} </strong>.</div>
@@ -70,9 +69,6 @@ const generateNotificationEmail = (
  ${moreRealEstates}
  </div>
  <br/>
-  ${wordOfMouthRequest}
-  <br/>
-  <br/>
  ${emailFooter}`;
 };

@@ -136,10 +132,6 @@ const generateNewSearchRequestEmail = (searchRequest, matchingRealEstates) => {
  </div>
  ${matchingRealEstates.length > 0 ? instantRealEstatesText : ""}
  <br/>
-  <br/>
-  ${wordOfMouthRequest}
-  <br/>
-  <br/>
  ${emailFooter}`;
 };

--- a/app/helpers/fetchWrapper.js
+++ b/app/helpers/fetchWrapper.js
@@ -1,58 +1,13 @@
 const nodeFetch = require("node-fetch");
-const AbortController = require('abort-controller');
-const FetchCache = require('@sozialhelden/fetch-cache').default;
-
-console.log("Fc ", FetchCache)
-
-const {
-  USER_AGENT,
-  USE_SCRAPER_API,
-  SCRAPER_API_KEY,
-  SCRAPER_API_BASE_URL,
-  NODE_FETCH_TIMEOUT_MS
-} = require("../config/appConfig");
-
-const timeout = (ms) => {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-const fetchCache = new FetchCache({
-  fetch: nodeFetch,
-  cacheOptions: {
-    // Don't save more than 100 responses in the cache. Allows infinite responses by default
-    maximalItemCount: 10000,
-    // When should the cache evict responses when its full?
-    evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
-    defaultTTL: 6 * 60 * 60 * 1000  // 6 hours
-    // ...see https://github.com/sozialhelden/hamster-cache for all possible options
-  },
-});
-
-
-const fetch = async (url, options = {}, useCache = true) => {
-  const controller = new AbortController();
+const { USER_AGENT } = require("../config/appConfig");

+const fetch = async (url, options = {}) => {
  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }
-
-  newOptions.signal = controller.signal;
-
- // newOptions["headers"]["User-Agent"] = USER_AGENT;
-
-  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
-  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
-    urlToFetchThroughAPI = url; 
-  }
-
-  const urlAdaptedForScraping = USE_SCRAPER_API
-    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
-    : url;
-  const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);
-
-  const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
-  return result;
+  newOptions["headers"]["User-Agent"] = USER_AGENT;
+  return nodeFetch(url, newOptions);
 };

 module.exports = fetch;
--- a/app/helpers/log.js
+++ b/app/helpers/log.js
@@ -1,13 +0,0 @@
-const {
-  PRINT_CRAWLER_DEBUG
-} = require("../config/appConfig");
-
-const logDebug = (...args) => {
-  if (PRINT_CRAWLER_DEBUG) {
-    console.log(...args);
-  }
-}
-
-module.exports = {
-  logDebug
-};
--- a/app/helpers/url.js
+++ b/app/helpers/url.js
@@ -7,26 +7,6 @@ const currentSearchRequest = async req => {

  return await getSearchRequest(searchRequestId);
 };
-
-const getUrlParams = function (url) {
-  if (typeof url === 'string' && url.length > 0){
-    const params = {};
-    const questionMarkIndex = url.indexOf('?');
-    if (questionMarkIndex === -1) {
-      return undefined;
-    }
-    const query = url.substring(questionMarkIndex+1);
-    const vars = query.split('&');
-    for (let i = 0; i < vars.length; i++) {
-      const pair = vars[i].split('=');
-      params[pair[0]] = decodeURIComponent(pair[1]);
-    }
-    return params;
-  }
-  return undefined;
-};
-
 module.exports = {
-  currentSearchRequest,
-  getUrlParams
+  currentSearchRequest
 };
--- a/app/migrations/20200215002846-add-column-notified-to-searchrequests.js
+++ b/app/migrations/20200215002846-add-column-notified-to-searchrequests.js
@@ -1,14 +0,0 @@
-"use strict";
-
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
-      type: Sequelize.DATE,
-      defaultValue: new Date()
-    });
-  },
-
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
-  }
-};
--- a/app/models/index.js
+++ b/app/models/index.js
@@ -16,7 +16,7 @@ config.logging = parseInt(process.env.SEQUELIZE_LOGGING) ? console.log : false;

 let sequelize;
 if (config.use_env_variable) {
-  sequelize = new Sequelize(process.env[config.use_env_variable] + "?ssl=true", config);
+  sequelize = new Sequelize(process.env[config.use_env_variable], config);
 } else {
  sequelize = new Sequelize(
    config.database,
--- a/app/models/searchRequest.js
+++ b/app/models/searchRequest.js
@@ -82,11 +82,7 @@ module.exports = (sequelize, DataTypes) => {
    floorMin: DataTypes.INTEGER,
    floorMax: DataTypes.INTEGER,
    accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT,
-    notifiedAt: {
-      type: DataTypes.DATE,
-      defaultValue: new Date()
-    }
+    heatingType: DataTypes.TEXT
  });

  return SearchRequest;
--- a/app/services/notificationService.js
+++ b/app/services/notificationService.js
@@ -15,10 +15,9 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
  findNotNotifiedMatches,
+  findAllRequestsForCheckUp,
  findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
-const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
-
 const { sendEmail } = require("../services/emailService");

 const notifyForNewRealEstates = async newRealEstates => {
@@ -36,7 +35,7 @@ const notifyForNewSearchRequest = async searchRequest => {
    matchingRealEstates
  );
  const { email } = searchRequest;
-  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
+
  await sendEmail(
    email,
    `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -77,10 +76,6 @@ const notifyMatches = async (matches, dailyNotification = false) => {
        sendEmailPromise.catch(err =>
          console.log("[Email Sending Failed]", err)
        );
-
-        //Change time of notified At for searchReq
-        searchRequest.notifiedAt = new Date();
-        searchRequest.save();
      }
    }
  }
@@ -148,12 +143,8 @@ const checkUpNotify = async () => {
    const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
    asyncSendEmailActions.push(sendEmailPromise);
    sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
-
-    //Change time of notified At for searchReq
-    searchRequest.notifiedAt = new Date();
-    searchRequest.save();
  }
-  await Promise.all(asyncSendEmailActions);*/
+  await Promise.all(asyncSendEmailActions); */
 };

 module.exports = {
--- a/development.env
+++ b/development.env
@@ -22,11 +22,6 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)

-#=============== SCRAPER API SUPORT =============#
-USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api
-SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
-
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)
@@ -72,7 +67,6 @@ AKTIDO_IGNORED_USERNAMES=!!! This is not used for aktido crawler !!!
 AKTIDO_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 AKTIDO_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
 #==SALJIC NEKRETNINE==
-SALJIC_MAX_PAGES=Restrict crawler to this number of pages
 SALJIC_MAX_RESULTS_PER_PAGE=For Saljic crawler, this represents how many ads are crawled at once
 SALJIC_CRAWLER_AD_TYPE=enum name of what type of ads should be crawled, check common/enums.js file for valid values
 SALJIC_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be included, check common/enums.js file for valid values
--- a/index.js
+++ b/index.js
@@ -4,7 +4,6 @@ const bodyParser = require("body-parser");
 const layout = require("express-layout");
 const compression = require("compression");
 const forceSSL = require("./app/helpers/forceSSL");
-const { logDebug } = require("./app/helpers/log");

 const {
  APP_PORT,
@@ -39,17 +38,11 @@ app.listen(APP_PORT, () =>

 let crawlerRunning = STOP_CRAWLER;
 const crawl = () => {
-  logDebug("Crawl start. crawlerRunning: ", crawlerRunning);
  if (!crawlerRunning) {
    crawlerRunning = true;
    crawlAll().then(newRealEstates => {
-      logDebug("crawlAll done, new real estate len: ", newRealEstates.length);
-      notifyForNewRealEstates(newRealEstates);
-    }).catch(e => {
-      console.error('Error happened: ', e);
-    }).finally(()=> {
      crawlerRunning = false;
-      logDebug('Finally done!');
+      notifyForNewRealEstates(newRealEstates);
    });
  }
 };
--- a/package-lock.json
+++ b/package-lock.json
@@ -40,32 +40,6 @@
        "@sendgrid/helpers": "^6.3.0"
      }
    },
-    "@sozialhelden/fetch-cache": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@sozialhelden/fetch-cache/-/fetch-cache-2.0.1.tgz",
-      "integrity": "sha512-vMlsdT5JQCGjx1fcFxmMNh7ZKppjjsfUAeZEhhNwhEL7GaqbZXsD1OXEyx2IcRa25ZuZtvJSV6Q3rE77VRdLvg==",
-      "requires": {
-        "@sozialhelden/hamster-cache": "^1.0.0"
-      }
-    },
-    "@sozialhelden/hamster-cache": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@sozialhelden/hamster-cache/-/hamster-cache-1.0.0.tgz",
-      "integrity": "sha512-/TEGA8mdMawZp4Yq/GrkL+72YL5EGuSeVXC3pKW12YY1t3C+zCN/HZ0HRp4zWF/e67svXcxuz/B0AEQxEdvi7A=="
-    },
-    "@supercharge/goodies": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/@supercharge/goodies/-/goodies-1.4.0.tgz",
-      "integrity": "sha512-Np6u2qjRwiA3wTgzz4n2yduydIjSXqtJWP5cOnNqjdlCR/EUAK86LAOhEcU+YW211D1ksugns3GqpARJDoXQ7g=="
-    },
-    "@supercharge/promise-pool": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-1.3.0.tgz",
-      "integrity": "sha512-9/EVrJevSPEqI4i/gRH8Dt7C+FQT65wRRYuu0MDaGmSLZ2aTel0jOGu8Ae84fPiQ+Ah0B80RPFUxk+K+Cz48DA==",
-      "requires": {
-        "@supercharge/goodies": "~1.4.0"
-      }
-    },
    "@types/caseless": {
      "version": "0.12.2",
      "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz",
@@ -105,14 +79,6 @@
      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
    },
-    "abort-controller": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
-      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
-      "requires": {
-        "event-target-shim": "^5.0.0"
-      }
-    },
    "accepts": {
      "version": "1.3.5",
      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz",
@@ -1108,11 +1074,6 @@
        "es5-ext": "~0.10.14"
      }
    },
-    "event-target-shim": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
-      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
-    },
    "events": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
--- a/package.json
+++ b/package.json
@@ -17,8 +17,7 @@
    "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
    "test-search": "cd test && node searchTest.js",
    "test-olx-scraper": "cd test && node olxScrapeTest.js",
-    "test-rental-scraper": "cd test && node rentalScrapeTest.js",
-    "test-saljic-scraper": "cd test && node saljicScrapeTest.js"
+    "test-rental-scraper": "cd test && node rentalScrapeTest.js"
  },
  "repository": {
    "type": "git",
@@ -32,9 +31,6 @@
  "dependencies": {
    "2checkout-node": "0.0.1",
    "@sendgrid/mail": "^6.3.1",
-    "@sozialhelden/fetch-cache": "^2.0.1",
-    "@supercharge/promise-pool": "^1.3.0",
-    "abort-controller": "^3.0.0",
    "aws-sdk": "^2.422.0",
    "bluebird": "^3.5.5",
    "cheerio": "^1.0.0-rc.2",
--- a/test/olxScrapeTest.js
+++ b/test/olxScrapeTest.js
@@ -9,7 +9,7 @@ if (urlToScrape) {

  (async () => {
    const data = await crawler.scrapeAd(urlToScrape);
-    console.log("Scraped data:", data);
+    console.log(data);
  })();
 } else {
  console.log("No URL to scrape. Use like this : ");
--- a/test/saljicScrapeTest.js
+++ b/test/saljicScrapeTest.js
@@ -1,17 +0,0 @@
-"use strict";
-
-const saljicCrawler = require("../app/crawler/specificCrawlers/saljic");
-
-const urlToScrape = process.argv[2] || undefined;
-
-if (urlToScrape) {
-  const crawler = new saljicCrawler();
-
-  (async () => {
-    const data = await crawler.scrapeAd(urlToScrape);
-    console.log("Scraped data:", data);
-  })();
-} else {
-  console.log("No URL to scrape. Use like this : ");
-  console.log("npm run test-saljic-scraper -- URL_TO_SCRAPE");
-}
Author	SHA1	Message	Date
Naida Vatric	d7fcb2a278	Merge branch 'master' after user-agent change into email-density	2020-02-21 14:26:33 +01:00
Naida Vatric	6bad24d735	New query for search req search.	2020-02-21 14:25:10 +01:00
Naida Vatric	7302edceec	Changed queries logic again.	2020-02-18 15:04:26 +01:00
Naida Vatric	bd33a6b80e	Logs for query check.	2020-02-17 23:24:55 +01:00