Olx added preflight check of available concurrent req.

Saljic crawler changed substring call.

See merge request saburly/marketalarm/web!95

app/config/appConfig.js

@@ -45,6 +45,11 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+const NUMBER_OF_CONCURRENT_REQ_SCRAPER_API =
+  parseInt(process.env.NUMBER_OF_CONCURRENT_REQ_SCRAPER_API) || 10;
+
 module.exports = {
   APP_PORT,
   APP_URL,
@@ -59,5 +64,8 @@ module.exports = {
   STAGING,
   CHECK_UP_DAYS,
   PROSTOR_LOGIN,
-  USER_AGENT
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
 };

app/crawler/specificCrawlers/olx.js

@@ -18,7 +18,9 @@ const {
 
 const {
   DEFAULT_TIMEZONE,
-  PRINT_CRAWLER_DEBUG
+  PRINT_CRAWLER_DEBUG,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API,
+  SCRAPER_API_KEY
 } = require("../../config/appConfig");
 
 const OLX_ENUMS = {
@@ -44,6 +46,8 @@ const OLX_ENUMS = {
 
 const { OLX_FORCE_CRAWL } = require("../specificConfigs/olx");
 
+const scraperapiClient = require("scraperapi-sdk")(SCRAPER_API_KEY);
+
 class OlxCrawler {
   constructor(
     savers = [],
@@ -190,12 +194,40 @@ class OlxCrawler {
       let actualNoOfResults =
         hrefs.length <= maxResultsPerPage ? hrefs.length : maxResultsPerPage;
 
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefs[i]));
+      const scrapedData = [];
+      for (
+        let i = 0;
+        i <= actualNoOfResults;
+        i = i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+      ) {
+        const concurrentUrlsToScrape = hrefs.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+        //Before it send n req to scraperAPI it send preflight request to check if we have enough concurrent req availabe
+        //It does not send "real" req until approven internaly
+        let availableConcurrentReqSlots = false;
+        do {
+          availableConcurrentReqSlots = await this.checkAvailableConcurrentReqSlots(
+            concurrentUrlsToScrape.length
+          );
+        } while (availableConcurrentReqSlots !== true);
+        //
+        console.log(
+          `OLX - Sending requests from ${i} to ${i +
+            NUMBER_OF_CONCURRENT_REQ_SCRAPER_API}.`
+        );
+        console.log(`OLX - Urls sent to scrape: `, concurrentUrlsToScrape);
+        //
+        const concurrentReqScraperApi = concurrentUrlsToScrape.map(url =>
+          this.scrapeAd(url)
+        );
+
+        const concurrentReqData = await Promise.all(concurrentReqScraperApi);
+
+        concurrentReqData.forEach(reqData => scrapedData.push(reqData));
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
       const filteredScrapedData = scrapedData.filter(adData => !!adData);
       return filteredScrapedData;
     } catch (e) {
@@ -206,6 +238,7 @@ class OlxCrawler {
 
   async scrapeAd(url) {
     // console.log("Scraping : ", url);
+
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
@@ -238,15 +271,28 @@ class OlxCrawler {
 
       //====== PRICE DETECTION AND EXTRACTION =====
       let price = null;
-      const normalPriceValue = $("#pc > p:nth-child(2)").text();
+      let normalPrice = null;
+      let urgentPrice = null;
+      const normalPriceValue = $("#pc > p:nth-child(2)")
+        .text()
+        .trim();
       const urgentPriceValue = $(
         "#artikal_glavni_div > div.artikal_lijevo > div:nth-child(5) > p"
       )
         .text()
         .trim();
+      //For cases where price is given in discount manner - different from default parsing
+      const discountPriceValue = $(
+        "#artikal_glavni_div > div.artikal_lijevo > div.op.pop > p"
+      )
+        .text()
+        .trim();
 
       if (normalPriceValue && normalPriceValue.length > 0) {
-        price = normalPriceValue;
+        normalPrice = normalPriceValue
+          .replace(/\r\n|\n|\r/gm, "")
+          .replace("KM", "")
+          .trim();
         if (
           $("#pc > p.n")
             .text()
@@ -256,21 +302,35 @@ class OlxCrawler {
         } else {
           status = AD_STATUS.STATUS_NORMAL;
         }
-      } else if (urgentPriceValue && urgentPriceValue.length > 0) {
-        const priceValues = urgentPriceValue.split("KM");
+      } else if (discountPriceValue && discountPriceValue.length > 0) {
+        status = AD_STATUS.STATUS_URGENT;
+        const priceValues = discountPriceValue.split("KM");
+        normalPrice = priceValues[0].trim();
+      } else {
+        console.log("Body:", body);
+        throw { message: "Can't find normal price" };
+      }
+      if (urgentPriceValue && urgentPriceValue.length > 0) {
+        const priceValues = urgentPriceValue.replace("Cijena", "").split("KM");
         //priceValues will contain values like ["100000", "90000", ...], second element is urgent price
-        if (priceValues.length > 1) {
-          price = priceValues[1].trim();
-          status = AD_STATUS.STATUS_DISCOUNTED;
+        if (priceValues.length > 0) {
+          if (priceValues[0].trim().indexOf("Hitno") != -1) {
+            urgentPrice = priceValues[0].replace("Hitno", "").trim();
+            status = AD_STATUS.STATUS_URGENT;
+          } else {
+            urgentPrice = priceValues[0].trim();
+          }
+        } else if (discountPriceValue && discountPriceValue.length > 0) {
+          status = AD_STATUS.STATUS_URGENT;
+          const priceValues = discountPriceValue.split("KM");
+          urgentPrice = priceValues[1].trim();
         } else {
           throw { message: "Can't find urgent price" };
         }
-      } else {
-        throw {
-          message: "Can't find price (it is not normal nor urgent price ?)"
-        };
       }
 
+      price = status === AD_STATUS.STATUS_URGENT ? urgentPrice : normalPrice;
+
       //====== OTHER AD INFORMATION ===============
       let adType = null;
       let olxId = null;
@@ -278,7 +338,7 @@ class OlxCrawler {
 
       let otherInformationDivId;
       //We need to locate DIV ID where other information are stored
-      for (let possibleId = 10; possibleId <= 20; possibleId++) {
+      for (let possibleId = 1; possibleId <= 30; possibleId++) {
         const adTypeFieldTitle = $(
           `#artikal_glavni_div > div.artikal_lijevo > div:nth-child(${possibleId}) > div:nth-child(2) > div.df1`
         )
@@ -655,6 +715,7 @@ class OlxCrawler {
     } catch (e) {
       console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
+
     return null;
   }
 
@@ -768,6 +829,9 @@ class OlxCrawler {
     if (!priceText) {
       return NaN;
     }
+    if (priceText === "Po dogovoru") {
+      return null;
+    }
     const formattedPriceText = priceText.replace(".", "").replace(",", ".");
     return parseFloat(formattedPriceText);
   }
@@ -867,8 +931,28 @@ class OlxCrawler {
     console.log("sprat = NEPOZNATO [", floorText, "]");
     return null;
   }
+  async checkAvailableConcurrentReqSlots(numberOfNeededConcurrentReq) {
+    try {
+      const scraperApiAccountInfo = await scraperapiClient.account();
+      const numberOfUsedConcurrentReq =
+        scraperApiAccountInfo.concurrentRequests;
+      const limitOfConcurrentReq = scraperApiAccountInfo.concurrencyLimit;
+      //Buffer of requests to prevent errors with prefligh requests
+      const bufferNumberOfReq = 3;
+      const numberOfAvailableConcurrentReq =
+        limitOfConcurrentReq - bufferNumberOfReq - numberOfUsedConcurrentReq;
+      if (numberOfNeededConcurrentReq <= numberOfAvailableConcurrentReq) {
+        return true;
+      } else {
+        return false;
+      }
+    } catch (err) {
+      return false;
+    }
+  }
 
   async sleep(ms) {
+    // console.log("Sleep for:", ms);
     return new Promise(resolve => setTimeout(resolve, ms));
   }
 

app/crawler/specificCrawlers/prostor.js

@@ -4,6 +4,7 @@ const fetch = require("../../helpers/fetchWrapper");
 const cheerio = require("cheerio");
 const moment = require("moment-timezone");
 const FormData = require("form-data");
+const nodeFetch = require("node-fetch");
 
 const {
   AD_TYPE,
@@ -197,7 +198,7 @@ class ProstorCrawler {
 
     // console.log("[PROSTOR] Scraping : ", url);
     try {
-      const adPageSource = await fetch(url, {
+      const adPageSource = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await adPageSource.text();
@@ -427,7 +428,7 @@ class ProstorCrawler {
     }
 
     try {
-      const res = await fetch(url, {
+      const res = await nodeFetch(url, {
         headers: { Cookie: prostorCookie }
       });
       const body = await res.text();
@@ -591,7 +592,7 @@ class ProstorCrawler {
     formData.append("email", PROSTOR_LOGIN.EMAIL);
     formData.append("password", PROSTOR_LOGIN.PASSWORD);
 
-    return fetch("https://prostor.ba/moj-prostor/prijava", {
+    return nodeFetch("https://prostor.ba/moj-prostor/prijava", {
       method: "POST",
       body: formData,
       headers: { Cookie: prostorCookie }
@@ -618,9 +619,12 @@ class ProstorCrawler {
       });
   }
   async getCookies() {
-    const getResponse = await fetch("https://prostor.ba/moj-prostor/prijava", {
-      headers: { Cookie: "" }
-    });
+    const getResponse = await nodeFetch(
+      "https://prostor.ba/moj-prostor/prijava",
+      {
+        headers: { Cookie: "" }
+      }
+    );
     const raw = getResponse.headers.raw()["set-cookie"];
     const cookie = raw
       .map(datastring => {

app/crawler/specificCrawlers/saljic.js

@@ -16,7 +16,8 @@ const {
 
 const {
   PRINT_CRAWLER_DEBUG,
-  DEFAULT_TIMEZONE
+  DEFAULT_TIMEZONE,
+  NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
 } = require("../../config/appConfig");
 const { SALJIC_FORCE_CRAWL } = require("../specificConfigs/saljic");
 
@@ -84,6 +85,7 @@ class SaljicCrawler {
         for (const [index, { value: singlePageResult }] of entries) {
           if (singlePageResult) {
             const saveResults = await this.saveCrawledResults(singlePageResult);
+
             const { newRecords } = saveResults;
 
             newRealEstates.push(...newRecords);
@@ -203,13 +205,32 @@ class SaljicCrawler {
           ? hrefsAbs.length
           : maxResultsPerPage;
 
-      const asyncScraping = [];
-      for (let i = 0; i < actualNoOfResults; i++) {
-        asyncScraping.push(this.scrapeAd(hrefsAbs[i], adTypes[i]));
+      const scrapedData = [];
+      for (
+        let i = 0;
+        i <= actualNoOfResults;
+        i = i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+      ) {
+        const concurrentUrlsToScrape = hrefsAbs.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+
+        const concurrentAdTypesOfReq = adTypes.slice(
+          i,
+          i + NUMBER_OF_CONCURRENT_REQ_SCRAPER_API
+        );
+
+        const concurrentReqScraperApi = concurrentUrlsToScrape.map(
+          (url, index) => this.scrapeAd(url, concurrentAdTypesOfReq[index])
+        );
+        const concurrentReqData = await Promise.all(concurrentReqScraperApi);
+
+        concurrentReqData.forEach(reqData => scrapedData.push(reqData));
       }
 
-      const scrapedData = await Promise.all(asyncScraping);
       const filteredScrapedData = scrapedData.filter(adData => !!adData);
+
       return filteredScrapedData;
     } catch (e) {
       console.error("[SALJIC] Exception caught:" + e);
@@ -217,22 +238,28 @@ class SaljicCrawler {
     }
   }
 
-  async scrapeAd(url, adType) {
-    // console.log("[SALJIC] Scraping : ", url);
+  async scrapeAd(url, adTypeAttribute) {
+    //console.log("[SALJIC] Scraping : ", url);
     try {
       const adPageSource = await fetch(url);
       const body = await adPageSource.text();
       const $ = cheerio.load(body);
 
+      //Throws error if req to Scraper API proxy wasn't succesful and responds with error
+      if (body.indexOf("<html>") === -1) {
+        throw { message: "Scraper API server error." };
+      }
       // No information for status ex. PRODAN
       const status = AD_STATUS.STATUS_NORMAL;
       //Extracting agency ID from url
-      const agencyObjectId = parseInt(url.substring(46, url.length));
+      const agencyObjectId = url
+        ? parseInt(url.substring(46, url.length))
+        : null;
 
       //Extracting main properties
       const propertySelectors = {
         title:
-          "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-title > h2",
+          "div.content-wrap > div.container.clearfix.wpc > div.col-md-8.nobottommargin > div.single-post.nobottommargin > div.entry.clearfix > div.entry-title > h2",
         price:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.topmargin-sm.single-product > div.product > div.product-price > ins",
         streetName:
@@ -243,6 +270,7 @@ class SaljicCrawler {
         latAndLong:
           "div.content-wrap > div.container > div.col-md-8.nobottommargin > div.single-post > div.entry > div.entry-content.topmargin > div.gmap.bottommargin > iframe"
       };
+
       const title = $(propertySelectors.title)
         .text()
         .replace(/(\r\n|\n|\r)/gm, "")
@@ -272,14 +300,15 @@ class SaljicCrawler {
         .trim();
 
       const latAndLongSrc = $(propertySelectors.latAndLong).attr("src");
-      const latText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("marker=") + 7,
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker="))
-      );
-      const longText = latAndLongSrc.substring(
-        latAndLongSrc.indexOf("%2C", latAndLongSrc.indexOf("marker=")) + 3,
-        latAndLongSrc.length
-      );
+      let tmpLatLong;
+      let latText;
+      let longText;
+
+      if (latAndLongSrc && latAndLongSrc.indexOf("openstreetmap") !== -1) {
+        tmpLatLong = latAndLongSrc.split("marker=")[1];
+        latText = tmpLatLong.split("%2C")[0];
+        longText = tmpLatLong.split("%2C")[1];
+      }
       const locationLat = parseFloat(latText) || null;
       const locationLong = parseFloat(longText) || null;
 
@@ -328,11 +357,11 @@ class SaljicCrawler {
       let numberOfViewsKivi = null;
       let streetNumber = 0;
       let adStatus = status;
-      let shortDescription = descriptions.substring(
-        0,
-        descriptions.indexOf(".")
-      );
-      let longDescription = descriptions;
+      let adType = adTypeAttribute;
+      let shortDescription = descriptions
+        ? descriptions.substring(0, descriptions.indexOf("."))
+        : "";
+      let longDescription = descriptions || "";
       //Extracting data - Glavne karakteristike
       let mainFieldIndex = 1;
       do {
@@ -343,10 +372,14 @@ class SaljicCrawler {
           .replace(/[\n\r\t]/gm, "")
           .trim();
 
-        const mainFieldTitle = mainField.substring(0, mainField.indexOf("  "));
+        const mainFieldTitle = mainField
+          ? mainField.substring(0, mainField.indexOf("  "))
+          : "";
         const mainFieldValue = mainField
-          .substring(mainField.indexOf("  "), mainField.length)
-          .trim();
+          ? mainField
+              .substring(mainField.indexOf("  "), mainField.length)
+              .trim()
+          : "";
 
         switch (mainFieldTitle) {
           case "Površina":
@@ -408,6 +441,7 @@ class SaljicCrawler {
               additionalField.length
             )
             .trim();
+
           realEstateType = this.getAdCategoryId(categoryTmp);
         } else {
           switch (additionalField) {
@@ -498,6 +532,11 @@ class SaljicCrawler {
       const region = "";
       const entity = "";
       const country = "";
+      //Throws error if realEstateType is null - not read. Still dont know why?
+      if (realEstateType === null) {
+        console.log("Body:", body);
+        throw { message: "Couldn't read real estate type." };
+      }
 
       const data = {
         url,
@@ -567,6 +606,7 @@ class SaljicCrawler {
     } catch (e) {
       console.error("Exception caught: " + e.message, "\r\nURL:", url);
     }
+
     return null;
   }
 

app/helpers/db/searchRequest.js

@@ -3,6 +3,7 @@ const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
 const { AD_CATEGORY } = require("../../common/enums");
+const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const getSearchRequest = async searchRequestId => {
   try {
@@ -16,6 +17,22 @@ const getSearchRequest = async searchRequestId => {
 const createSearchRequest = async (searchRequestFields = {}) => {
   return await db.SearchRequest.create(searchRequestFields);
 };
+const findAllRequestsForCheckUp = async () => {
+  const checkUpOffset = 24 * 60 * 60 * 1000 * CHECK_UP_DAYS; //in miliseconds
+  const checkupDate = new Date();
+  checkupDate.setTime(checkupDate.getTime() - checkUpOffset);
+
+  const dateQuery = {
+    notifiedAt: {
+      [Op.lte]: checkupDate
+    }
+  };
+  const allRequestsForCheckUp = await db.SearchRequest.findAll({
+    where: dateQuery
+  });
+
+  return allRequestsForCheckUp;
+};
 
 const findSearchRequestsForRealEstate = async realEstate => {
   const {
@@ -459,5 +476,6 @@ const findSearchRequestsForRealEstate = async realEstate => {
 module.exports = {
   getSearchRequest,
   createSearchRequest,
-  findSearchRequestsForRealEstate
+  findSearchRequestsForRealEstate,
+  findAllRequestsForCheckUp
 };

app/helpers/db/searchRequestMatch.js

@@ -2,7 +2,6 @@
 const db = require("../../models/index");
 const sequelize = require("sequelize");
 const Op = sequelize.Op;
-const { CHECK_UP_DAYS } = require("../../config/appConfig");
 
 const findRealEstatesForSearchRequest = async searchRequestId => {
   const query = {
@@ -43,42 +42,6 @@ const findNotNotifiedMatches = async () => {
 
   return matchingRecords;
 };
-const findAllRequestsForCheckUp = async () => {
-  //First we find IDs of search request that don't need to be emailed for check up - to EXCLUDE
-  //The ones that received notification for real estate CHECK_UP_DAYS days from now
-  const date = new Date();
-  const checkUpDate = date.getDate() - CHECK_UP_DAYS;
-  date.setDate(checkUpDate);
-  const dateQuery = {
-    createdAt: {
-      [Op.gte]: date
-    }
-  };
-
-  const excludedMatches = await db.SearchRequestMatch.findAll({
-    attributes: ["searchRequestId"],
-    where: dateQuery,
-    order: [["searchRequestId", "ASC"]]
-  });
-
-  const excludedRequestsAll = excludedMatches.map(match => {
-    return match.dataValues.searchRequestId;
-  });
-  //Removing duplicate search request id-s for optimization
-  const excludedRequests = [...new Set(excludedRequestsAll)];
-
-  const query = {
-    subscribed: true,
-    id: {
-      [Op.notIn]: excludedRequests
-    }
-  };
-  const allRequestsForCheckUp = await db.SearchRequest.findAll({
-    where: query
-  });
-
-  return allRequestsForCheckUp;
-};
 
 const addMatches = async matchingRecords => {
   return await db.SearchRequestMatch.bulkCreate(matchingRecords, {
@@ -89,6 +52,5 @@ const addMatches = async matchingRecords => {
 module.exports = {
   findRealEstatesForSearchRequest,
   addMatches,
-  findNotNotifiedMatches,
-  findAllRequestsForCheckUp
+  findNotNotifiedMatches
 };

app/helpers/fetchWrapper.js

@@ -1,5 +1,9 @@
 const nodeFetch = require("node-fetch");
-const { USER_AGENT } = require("../config/appConfig");
+const {
+  USER_AGENT,
+  USE_SCRAPER_API,
+  SCRAPER_API_KEY
+} = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
   const newOptions = Object.assign({}, options);
@@ -7,7 +11,14 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
-  return nodeFetch(url, newOptions);
+  const urlAdaptedForScraping = USE_SCRAPER_API
+    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    : url;
+
+  //
+  // console.log("Url for scraping:", urlAdaptedForScraping);
+
+  return nodeFetch(urlAdaptedForScraping, newOptions);
 };
 
 module.exports = fetch;

app/migrations/20200215002846-add-column-notified-to-searchrequests.js

@@ -0,0 +1,14 @@
+"use strict";
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.addColumn("SearchRequests", "notifiedAt", {
+      type: Sequelize.DATE,
+      defaultValue: new Date()
+    });
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.removeColumn("SearchRequests", "notifiedAt");
+  }
+};

app/models/searchRequest.js

@@ -82,7 +82,11 @@ module.exports = (sequelize, DataTypes) => {
     floorMin: DataTypes.INTEGER,
     floorMax: DataTypes.INTEGER,
     accessRoadType: DataTypes.TEXT,
-    heatingType: DataTypes.TEXT
+    heatingType: DataTypes.TEXT,
+    notifiedAt: {
+      type: DataTypes.DATE,
+      defaultValue: new Date()
+    }
   });
 
   return SearchRequest;

app/services/notificationService.js

@@ -15,9 +15,10 @@ const {
 } = require("../helpers/emailContentGenerator");
 const {
   findNotNotifiedMatches,
-  findAllRequestsForCheckUp,
   findRealEstatesForSearchRequest
 } = require("../helpers/db/searchRequestMatch");
+const { findAllRequestsForCheckUp } = require("../helpers/db/searchRequest");
+
 const { sendEmail } = require("../services/emailService");
 
 const notifyForNewRealEstates = async newRealEstates => {
@@ -35,7 +36,7 @@ const notifyForNewSearchRequest = async searchRequest => {
     matchingRealEstates
   );
   const { email } = searchRequest;
-
+  //In case of the new search req, notifiedAt column is populated with default value - now (moment of creation)
   await sendEmail(
     email,
     `${stagingTag} Kivi - novi zahtjev za pretragu`,
@@ -76,6 +77,10 @@ const notifyMatches = async (matches, dailyNotification = false) => {
         sendEmailPromise.catch(err =>
           console.log("[Email Sending Failed]", err)
         );
+
+        //Change time of notified At for searchReq
+        searchRequest.notifiedAt = new Date();
+        searchRequest.save();
       }
     }
   }
@@ -143,8 +148,12 @@ const checkUpNotify = async () => {
     const sendEmailPromise = sendEmail(email, emailSubject, emailContent);
     asyncSendEmailActions.push(sendEmailPromise);
     sendEmailPromise.catch(err => console.log("[Email Sending Failed]", err));
+
+    //Change time of notified At for searchReq
+    searchRequest.notifiedAt = new Date();
+    searchRequest.save();
   }
-  await Promise.all(asyncSendEmailActions); */
+  await Promise.all(asyncSendEmailActions);*/
 };
 
 module.exports = {

development.env

@@ -22,6 +22,11 @@ GA_ID=Google Analytics ID
 #=============== GOOGLE MAPS =============#
 API_MAP_KEY=(your-key-here)
 
+#=============== SCRAPER API SUPORT =============#
+USE_SCRAPER_API= To turn it on (1) or off (0)
+SCRAPER_API_KEY= Key for Scraper api 
+NUMBER_OF_CONCURRENT_REQ_SCRAPER_API= Number of requests to send concurrently to Srcaper API proxy
+
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
 AWS_SECRET_ACCESS_KEY=(your-key-here)
@@ -32,6 +37,7 @@ SOURCE_EMAIL=info@saburly.com
 CRAWLER_INTERVAL=Interval to run cralwer(s), in seconds
 STOP_CRAWLER=Non-zero value will skip crawler execution
 PRINT_CRAWLER_DEBUG_INFO=Non-zero value will print crawler debugging info to the server console
+
 #==OLX==
 OLX_MAX_PAGES=Restrict crawler to this number of pages
 OLX_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved
@@ -40,6 +46,7 @@ OLX_CRAWLER_AD_CATEGORIES=comma separated list of enum names of categories to be
 OLX_IGNORED_USERNAMES=comma separated list of usernames to ignore
 OLX_DELAY_BETWEEN_PAGES=time in miliseconds to wait before indexing next page
 OLX_FORCE_CRAWL=Non-zero value will force crawler to crawl all pages without stopping when known real estate is found
+
 #==RENTAL==
 RENTAL_MAX_PAGES=Restrict crawler to this number of pages
 RENTAL_MAX_RESULTS_PER_PAGE=Only this number or less results from one page will be scraped and saved

package-lock.json

@@ -147,6 +147,14 @@
         }
       }
     },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
     "arr-diff": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
@@ -195,6 +203,21 @@
       "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=",
       "dev": true
     },
+    "async": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
+      "requires": {
+        "lodash": "^4.17.14"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "4.17.15",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
+        }
+      }
+    },
     "async-each": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz",
@@ -625,6 +648,11 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
       "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
     },
+    "colors": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
+      "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs="
+    },
     "combined-stream": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
@@ -730,6 +758,25 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "coveralls": {
+      "version": "3.0.9",
+      "resolved": "https://registry.npmjs.org/coveralls/-/coveralls-3.0.9.tgz",
+      "integrity": "sha512-nNBg3B1+4iDox5A5zqHKzUTiwl2ey4k2o0NEcVZYvl+GOSJdKBj4AJGKLv6h3SvWch7tABHePAQOSZWM9E2hMg==",
+      "requires": {
+        "js-yaml": "^3.13.1",
+        "lcov-parse": "^1.0.0",
+        "log-driver": "^1.2.7",
+        "minimist": "^1.2.0",
+        "request": "^2.88.0"
+      },
+      "dependencies": {
+        "minimist": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
+          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
+        }
+      }
+    },
     "create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -782,6 +829,11 @@
       "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
       "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
     },
+    "cycle": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz",
+      "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI="
+    },
     "d": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
@@ -1060,6 +1112,11 @@
         "prettier-linter-helpers": "^1.0.0"
       }
     },
+    "esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="
+    },
     "etag": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
@@ -1274,6 +1331,11 @@
       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
       "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
     },
+    "eyes": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz",
+      "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A="
+    },
     "fast-deep-equal": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
@@ -2475,6 +2537,15 @@
         "nopt": "~4.0.1"
       }
     },
+    "js-yaml": {
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
+      "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
+      "requires": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      }
+    },
     "jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -2537,6 +2608,11 @@
         "invert-kv": "^2.0.0"
       }
     },
+    "lcov-parse": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz",
+      "integrity": "sha1-6w1GtUER68VhrLTECO+TY73I9+A="
+    },
     "locate-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
@@ -2551,6 +2627,11 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
       "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
     },
+    "log-driver": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
+      "integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg=="
+    },
     "long-timeout": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
@@ -3221,6 +3302,20 @@
       "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
       "dev": true
     },
+    "promise-request-retry": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/promise-request-retry/-/promise-request-retry-1.0.2.tgz",
+      "integrity": "sha512-zZmu19chRtC6TYeAZaELF8s+Zotl48M6bRnIVjcUrObEjpI4wk+2VpGVRaRgCG6isOqsK4c5IMY7t59Ff2ia0A==",
+      "requires": {
+        "async": "^2.6.0",
+        "bluebird": "^3.5.1",
+        "coveralls": "^3.0.0",
+        "req-cwd": "^2.0.0",
+        "request": "^2.85.0",
+        "request-promise": "^4.2.2",
+        "winston": "^2.4.0"
+      }
+    },
     "proto-list": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
@@ -3415,6 +3510,22 @@
       "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=",
       "dev": true
     },
+    "req-cwd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/req-cwd/-/req-cwd-2.0.0.tgz",
+      "integrity": "sha1-1AgrTURZgDZkD7c93qAe1T20nrw=",
+      "requires": {
+        "req-from": "^2.0.0"
+      }
+    },
+    "req-from": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/req-from/-/req-from-2.0.0.tgz",
+      "integrity": "sha1-10GI5H+TeW9Kpx327jWuaJ8+DnA=",
+      "requires": {
+        "resolve-from": "^3.0.0"
+      }
+    },
     "request": {
       "version": "2.88.0",
       "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",
@@ -3454,6 +3565,32 @@
         }
       }
     },
+    "request-promise": {
+      "version": "4.2.5",
+      "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.5.tgz",
+      "integrity": "sha512-ZgnepCykFdmpq86fKGwqntyTiUrHycALuGggpyCZwMvGaZWgxW6yagT0FHkgo5LzYvOaCNvxYwWYIjevSH1EDg==",
+      "requires": {
+        "bluebird": "^3.5.0",
+        "request-promise-core": "1.1.3",
+        "stealthy-require": "^1.1.1",
+        "tough-cookie": "^2.3.3"
+      }
+    },
+    "request-promise-core": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.3.tgz",
+      "integrity": "sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==",
+      "requires": {
+        "lodash": "^4.17.15"
+      },
+      "dependencies": {
+        "lodash": {
+          "version": "4.17.15",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
+        }
+      }
+    },
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@@ -3472,6 +3609,11 @@
         "path-parse": "^1.0.6"
       }
     },
+    "resolve-from": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
+      "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g="
+    },
     "resolve-url": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
@@ -3516,6 +3658,16 @@
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
       "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
     },
+    "scraperapi-sdk": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/scraperapi-sdk/-/scraperapi-sdk-1.0.3.tgz",
+      "integrity": "sha512-wFzdVptJHAA13HWMxR6DxsesA95cx0eBvylh2CHH9UmzBYor7N54jxgL473IW1VZEferSCNpwlW2R/B3zTPDsQ==",
+      "requires": {
+        "promise-request-retry": "^1.0.2",
+        "request": "^2.88.0",
+        "request-promise": "^4.2.5"
+      }
+    },
     "semver": {
       "version": "5.6.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
@@ -3838,6 +3990,11 @@
         "extend-shallow": "^3.0.0"
       }
     },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
+    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@@ -3854,6 +4011,11 @@
         "tweetnacl": "~0.14.0"
       }
     },
+    "stack-trace": {
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
+      "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
+    },
     "static-extend": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
@@ -3880,6 +4042,11 @@
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
       "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
     },
+    "stealthy-require": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
+      "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
+    },
     "string-width": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
@@ -4351,6 +4518,26 @@
         "string-width": "^2.1.1"
       }
     },
+    "winston": {
+      "version": "2.4.4",
+      "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz",
+      "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==",
+      "requires": {
+        "async": "~1.0.0",
+        "colors": "1.0.x",
+        "cycle": "1.0.x",
+        "eyes": "0.1.x",
+        "isstream": "0.1.x",
+        "stack-trace": "0.0.x"
+      },
+      "dependencies": {
+        "async": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz",
+          "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k="
+        }
+      }
+    },
     "wkx": {
       "version": "0.4.8",
       "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",

package.json

@@ -17,7 +17,9 @@
     "checkup-notify": "cd app/npmScripts && node npmCheckUpNotify.js",
     "test-search": "cd test && node searchTest.js",
     "test-olx-scraper": "cd test && node olxScrapeTest.js",
-    "test-rental-scraper": "cd test && node rentalScrapeTest.js"
+    "test-saljic-scraper": "cd test && node saljicScrapeTest.js",
+    "test-rental-scraper": "cd test && node rentalScrapeTest.js",
+    "test-scraper-api": "cd test && node scraperAPITest.js"
   },
   "repository": {
     "type": "git",
@@ -50,6 +52,7 @@
     "pg": "^7.10.0",
     "prettier": "^1.19.1",
     "react-step-wizard": "^5.1.0",
+    "scraperapi-sdk": "^1.0.3",
     "sequelize": "^5.18.4",
     "sequelize-cli": "^5.5.0"
   },

test/olxScrapeTest.js

@@ -9,7 +9,7 @@ if (urlToScrape) {
 
   (async () => {
     const data = await crawler.scrapeAd(urlToScrape);
-    console.log(data);
+    console.log("Scraped data:", data);
   })();
 } else {
   console.log("No URL to scrape. Use like this : ");

test/saljicScrapeTest.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const saljicCrawler = require("../app/crawler/specificCrawlers/saljic");
+
+const urlToScrape = process.argv[2] || undefined;
+
+if (urlToScrape) {
+  const crawler = new saljicCrawler();
+
+  (async () => {
+    const data = await crawler.scrapeAd(urlToScrape);
+    console.log("Scraped data:", data);
+  })();
+} else {
+  console.log("No URL to scrape. Use like this : ");
+  console.log("npm run test-saljic-scraper -- URL_TO_SCRAPE");
+}

test/scraperAPITest.js

@@ -0,0 +1,19 @@
+const { SCRAPER_API_KEY } = require("../app/config/appConfig");
+
+const scraperapiClient = require("scraperapi-sdk")(SCRAPER_API_KEY);
+
+async function logUsedConcurrentReq() {
+  try {
+    const response = await scraperapiClient.account();
+    const dateOfLog = new Date().toLocaleString();
+    console.log(
+      dateOfLog,
+      " Number of concurrent requests: ",
+      response.concurrentRequests
+    );
+  } catch (err) {
+    console.log(err.message);
+  }
+}
+
+setInterval(logUsedConcurrentReq, 1000);