senaduka/old-web
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Change for scraperapi

Browse Source
This commit is contained in:
Senad Uka
2020-09-10 17:34:39 +02:00
parent ecf27f2ba1
commit 78c6056db4
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/helpers/fetchWrapper.js
View File

@@ -13,8 +13,13 @@ const fetch = async (url, options = {}) => {
} }
newOptions["headers"]["User-Agent"] = USER_AGENT; newOptions["headers"]["User-Agent"] = USER_AGENT;
let urlToFetchThroughAPI = Buffer.from(url).toString('base64');
if (SCRAPER_API_BASE_URL.includes('scraperapi')) {
urlToFetchThroughAPI = url;
}
const urlAdaptedForScraping = USE_SCRAPER_API const urlAdaptedForScraping = USE_SCRAPER_API
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}` ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
: url; : url;
return nodeFetch(urlAdaptedForScraping, newOptions); return nodeFetch(urlAdaptedForScraping, newOptions);