From d436d4a37bd73a4dad5f25e47b685d1183aaf120 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Sat, 22 Feb 2020 22:15:27 +0100 Subject: [PATCH 1/3] Added Scraper API option. --- app/config/appConfig.js | 7 ++++++- app/helpers/fetchWrapper.js | 12 ++++++++++-- development.env | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/app/config/appConfig.js b/app/config/appConfig.js index 411e6ff..2843248 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -45,6 +45,9 @@ const USER_AGENT = process.env.USER_AGENT || "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"; +const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 0; //Default not to use? +const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || ""; + module.exports = { APP_PORT, APP_URL, @@ -59,5 +62,7 @@ module.exports = { STAGING, CHECK_UP_DAYS, PROSTOR_LOGIN, - USER_AGENT + USER_AGENT, + USE_SCRAPER_API, + SCRAPER_API_KEY }; diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js index 1d9b198..74e33c6 100644 --- a/app/helpers/fetchWrapper.js +++ b/app/helpers/fetchWrapper.js @@ -1,5 +1,9 @@ const nodeFetch = require("node-fetch"); -const { USER_AGENT } = require("../config/appConfig"); +const { + USER_AGENT, + USE_SCRAPER_API, + SCRAPER_API_KEY +} = require("../config/appConfig"); const fetch = async (url, options = {}) => { const newOptions = Object.assign({}, options); @@ -7,7 +11,11 @@ const fetch = async (url, options = {}) => { newOptions["headers"] = {}; } newOptions["headers"]["User-Agent"] = USER_AGENT; - return nodeFetch(url, newOptions); + const newUrl = USE_SCRAPER_API + ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}` + : url; + + return nodeFetch(newUrl, newOptions); }; module.exports = fetch; diff --git a/development.env b/development.env index bc1cc92..42844a6 100644 --- a/development.env +++ b/development.env @@ -22,6 +22,10 @@ GA_ID=Google Analytics ID #=============== GOOGLE MAPS =============# API_MAP_KEY=(your-key-here) +#=============== SCRAPER API SUPORT =============# +USE_SCRAPER_API= To turn it on (1) or off (0) +SCRAPER_API_KEY= Key for Scraper api + #=============== AWS SDK EMAIL SETTINGS =======# AWS_KEY_ID=(your-key-here) AWS_SECRET_ACCESS_KEY=(your-key-here) -- 2.47.3 From 148b2ea863178267f97a45d8d125001ac1d35a71 Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Sun, 23 Feb 2020 16:38:40 +0100 Subject: [PATCH 2/3] Changed default. --- app/config/appConfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config/appConfig.js b/app/config/appConfig.js index 2843248..0ff991a 100644 --- a/app/config/appConfig.js +++ b/app/config/appConfig.js @@ -45,7 +45,7 @@ const USER_AGENT = process.env.USER_AGENT || "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"; -const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 0; //Default not to use? +const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || ""; module.exports = { -- 2.47.3 From ce857ddce9c6006fac81cdcc16a19178b3762a9f Mon Sep 17 00:00:00 2001 From: Naida Vatric Date: Sun, 23 Feb 2020 23:11:21 +0100 Subject: [PATCH 3/3] Renamed var. --- app/helpers/fetchWrapper.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js index 74e33c6..3af2817 100644 --- a/app/helpers/fetchWrapper.js +++ b/app/helpers/fetchWrapper.js @@ -11,11 +11,11 @@ const fetch = async (url, options = {}) => { newOptions["headers"] = {}; } newOptions["headers"]["User-Agent"] = USER_AGENT; - const newUrl = USE_SCRAPER_API + const urlAdaptedForScraping = USE_SCRAPER_API ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}` : url; - return nodeFetch(newUrl, newOptions); + return nodeFetch(urlAdaptedForScraping, newOptions); }; module.exports = fetch; -- 2.47.3