diff --git a/app/helpers/fetchWrapper.js b/app/helpers/fetchWrapper.js index c303ddf..5fc6a81 100644 --- a/app/helpers/fetchWrapper.js +++ b/app/helpers/fetchWrapper.js @@ -13,8 +13,13 @@ const fetch = async (url, options = {}) => { } newOptions["headers"]["User-Agent"] = USER_AGENT; + let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); + if (SCRAPER_API_BASE_URL.includes('scraperapi')) { + urlToFetchThroughAPI = url; + } + const urlAdaptedForScraping = USE_SCRAPER_API - ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}` + ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}` : url; return nodeFetch(urlAdaptedForScraping, newOptions);