senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Merge branch 'move-scraper-url-to-the-env' into 'master'

Browse Source 
Move scraper api url to the ENV

See merge request saburly/marketalarm/web!108
This commit was merged in pull request #108.
This commit is contained in:
Senad Uka
2020-05-12 11:55:12 +00:00
parent 692577fb8c ff923605ad
commit f1d45fed26
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/config/appConfig.js
View File

@@ -45,8 +45,9 @@ const USER_AGENT =
process.env.USER_AGENT ||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36";
const USE_SCRAPER_API = process.env.USE_SCRAPER_API || 1; //Default to use
const USE_SCRAPER_API = process.env.USE_SCRAPER_API === undefined ? 1 : parseInt(process.env.USE_SCRAPER_API);
const SCRAPER_API_KEY = process.env.SCRAPER_API_KEY || "";
const SCRAPER_API_BASE_URL = process.env.SCRAPER_API_BASE_URL || "";
module.exports = {
APP_PORT,
@@ -64,5 +65,6 @@ module.exports = {
PROSTOR_LOGIN,
USER_AGENT,
USE_SCRAPER_API,
SCRAPER_API_KEY
SCRAPER_API_KEY,
SCRAPER_API_BASE_URL
};

6
app/helpers/fetchWrapper.js
View File

@@ -2,7 +2,8 @@ const nodeFetch = require("node-fetch");
const {
USER_AGENT,
USE_SCRAPER_API,
SCRAPER_API_KEY
SCRAPER_API_KEY,
SCRAPER_API_BASE_URL
} = require("../config/appConfig");
const fetch = async (url, options = {}) => {
@@ -11,8 +12,9 @@ const fetch = async (url, options = {}) => {
newOptions["headers"] = {};
}
newOptions["headers"]["User-Agent"] = USER_AGENT;
const urlAdaptedForScraping = USE_SCRAPER_API
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
: url;
return nodeFetch(urlAdaptedForScraping, newOptions);

3
development.env
View File

@@ -24,7 +24,8 @@ API_MAP_KEY=(your-key-here)
#=============== SCRAPER API SUPORT =============#
USE_SCRAPER_API= To turn it on (1) or off (0)
SCRAPER_API_KEY= Key for Scraper api
SCRAPER_API_KEY= Key for Scraper api
SCRAPER_API_BASE_URL= Base url without question mark (example: http://sabur.kivi.ba:1337)
#=============== AWS SDK EMAIL SETTINGS =======#
AWS_KEY_ID=(your-key-here)