senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'move-scraper-url-to-the-env' into 'master'

Browse Source 
Move scraper api url to the ENV

See merge request saburly/marketalarm/web!108
This commit was merged in pull request #108.
This commit is contained in:
Senad Uka
2020-05-12 11:55:12 +00:00
parent 692577fb8c ff923605ad
commit f1d45fed26
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/config/appConfig.js
View File

@@ -45,8 +45,9 @@ const USER_AGENT =
process.env.USER_AGENT || process.env.USER_AGENT ||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"; "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || ""; const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
module.exports = { module.exports = {
APP_PORT, APP_PORT,
@@ -64,5 +65,6 @@ module.exports = {
PROSTOR_LOGIN, PROSTOR_LOGIN,
USER_AGENT, USER_AGENT,
USE_SCRAPER_API, USE_SCRAPER_API,
SCRAPER_API_KEY SCRAPER_API_KEY,
SCRAPER_API_BASE_URL
}; };

6
app/helpers/fetchWrapper.js
View File

@@ -2,7 +2,8 @@ const nodeFetch = require("node-fetch");
const { const {
USER_AGENT, USER_AGENT,
USE_SCRAPER_API, USE_SCRAPER_API,
SCRAPER_API_KEY SCRAPER_API_KEY,
SCRAPER_API_BASE_URL
} = require("../config/appConfig"); } = require("../config/appConfig");
const fetch = async (url, options = {}) => { const fetch = async (url, options = {}) => {
@@ -11,8 +12,9 @@ const fetch = async (url, options = {}) => {
newOptions["headers"] = {}; newOptions["headers"] = {};
} }
newOptions["headers"]["User-Agent"] = USER_AGENT; newOptions["headers"]["User-Agent"] = USER_AGENT;
const urlAdaptedForScraping = USE_SCRAPER_API const urlAdaptedForScraping = USE_SCRAPER_API
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}` ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
: url; : url;
return nodeFetch(urlAdaptedForScraping, newOptions); return nodeFetch(urlAdaptedForScraping, newOptions);

3
development.env
View File

@@ -24,7 +24,8 @@ API_MAP_KEY=(your-key-here)
#=============== SCRAPER API SUPORT =============# #=============== SCRAPER API SUPORT =============#
USE_SCRAPER_API= To turn it on (1) or off (0) USE_SCRAPER_API= To turn it on (1) or off (0)
SCRAPER_API_KEY= Key for Scraper api SCRAPER_API_KEY= Key for Scraper api
SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
#=============== AWS SDK EMAIL SETTINGS =======# #=============== AWS SDK EMAIL SETTINGS =======#
AWS_KEY_ID=(your-key-here) AWS_KEY_ID=(your-key-here)