app/config/appConfig.js

@@ -45,8 +45,9 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
-const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
 const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
 
 module.exports = {
   APP_PORT,
@@ -64,5 +65,6 @@ module.exports = {
   PROSTOR_LOGIN,
   USER_AGENT,
   USE_SCRAPER_API,
-  SCRAPER_API_KEY
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL
 };

app/helpers/fetchWrapper.js

@@ -2,7 +2,8 @@ const nodeFetch = require("node-fetch");
 const {
   USER_AGENT,
   USE_SCRAPER_API,
-  SCRAPER_API_KEY
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL
 } = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
@@ -11,8 +12,9 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
+
   const urlAdaptedForScraping = USE_SCRAPER_API
-    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
     : url;
 
   return nodeFetch(urlAdaptedForScraping, newOptions);

development.env

@@ -24,7 +24,8 @@ API_MAP_KEY=(your-key-here)
 
 #=============== SCRAPER API SUPORT =============#
 USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api 
+SCRAPER_API_KEY= Key for Scraper api
+SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)