From ff923605adae13ea420bd95316ba0c3bcedadd99 Mon Sep 17 00:00:00 2001
From: Bilal <bilal@saburly.com>
Date: Tue, 12 May 2020 13:44:09 +0200
Subject: [PATCH] Move scraper api base url to the ENV; send URL as base64
 string

---
 app/config/appConfig.js     | 6 ++++--
 app/helpers/fetchWrapper.js | 6 ++++--
 development.env             | 3 ++-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/app/config/appConfig.js b/app/config/appConfig.js
index 0ff991a..7ed2ada 100644
--- a/app/config/appConfig.js
+++ b/app/config/appConfig.js
@@ -45,8 +45,9 @@ const USER_AGENT =
   process.env.USER_AGENT ||
   "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
 
-const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
+const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
 const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
+const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
 
 module.exports = {
   APP_PORT,
@@ -64,5 +65,6 @@ module.exports = {
   PROSTOR_LOGIN,
   USER_AGENT,
   USE_SCRAPER_API,
-  SCRAPER_API_KEY
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL
 };
diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js
index 3af2817..c303ddf 100644
--- a/app/helpers/fetchWrapper.js
+++ b/app/helpers/fetchWrapper.js
@@ -2,7 +2,8 @@ const nodeFetch = require("node-fetch");
 const {
   USER_AGENT,
   USE_SCRAPER_API,
-  SCRAPER_API_KEY
+  SCRAPER_API_KEY,
+  SCRAPER_API_BASE_URL
 } = require("../config/appConfig");
 
 const fetch = async (url, options = {}) => {
@@ -11,8 +12,9 @@ const fetch = async (url, options = {}) => {
     newOptions["headers"] = {};
   }
   newOptions["headers"]["User-Agent"] = USER_AGENT;
+
   const urlAdaptedForScraping = USE_SCRAPER_API
-    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
+    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
     : url;
 
   return nodeFetch(urlAdaptedForScraping, newOptions);
diff --git a/development.env b/development.env
index 42844a6..f4a2588 100644
--- a/development.env
+++ b/development.env
@@ -24,7 +24,8 @@ API_MAP_KEY=(your-key-here)
 
 #=============== SCRAPER API SUPORT =============#
 USE_SCRAPER_API= To turn it on (1) or off (0)
-SCRAPER_API_KEY= Key for Scraper api 
+SCRAPER_API_KEY= Key for Scraper api
+SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
 
 #=============== AWS SDK EMAIL SETTINGS =======#
 AWS_KEY_ID=(your-key-here)
-- 
2.47.3