senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Move scraper api base url to the ENV; send URL as base64 string

Browse Source
This commit is contained in:
Bilal
2020-05-12 13:44:09 +02:00
parent 692577fb8c
commit ff923605ad
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/helpers/fetchWrapper.js
View File

@@ -2,7 +2,8 @@ const nodeFetch = require("node-fetch");
const {
USER_AGENT,
USE_SCRAPER_API,
SCRAPER_API_KEY
SCRAPER_API_KEY,
SCRAPER_API_BASE_URL
} = require("../config/appConfig");
const fetch = async (url, options = {}) => {
@@ -11,8 +12,9 @@ const fetch = async (url, options = {}) => {
newOptions["headers"] = {};
}
newOptions["headers"]["User-Agent"] = USER_AGENT;
const urlAdaptedForScraping = USE_SCRAPER_API
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
: url;
return nodeFetch(urlAdaptedForScraping, newOptions);