From 78c6056db4a15be2976df2061a1c6b85eb8486e7 Mon Sep 17 00:00:00 2001 From: Senad Uka Date: Thu, 10 Sep 2020 17:34:39 +0200 Subject: [PATCH] Change for scraperapi --- app/helpers/fetchWrapper.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js index c303ddf..5fc6a81 100644 --- a/app/helpers/fetchWrapper.js +++ b/app/helpers/fetchWrapper.js @@ -13,8 +13,13 @@ const fetch = async (url, options = {}) => { } newOptions["headers"]["User-Agent"] = USER_AGENT; + let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); + if (SCRAPER_API_BASE_URL.includes('scraperapi')) { + urlToFetchThroughAPI = url; + } + const urlAdaptedForScraping = USE_SCRAPER_API - ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}` + ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}` : url; return nodeFetch(urlAdaptedForScraping, newOptions);