old-web/app/helpers/fetchWrapper.js

const nodeFetch = require("node-fetch");
const AbortController = require('abort-controller');
const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY,
  SCRAPER_API_BASE_URL
} = require("../config/appConfig");

const timeout = (ms) => {
    return new Promise(resolve => setTimeout(resolve, ms));
}

const fetch = async (url, options = {}) => {
  const controller = new AbortController();

  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }

  newOptions.signal = controller.signal;

 // newOptions["headers"]["User-Agent"] = USER_AGENT;

  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
    urlToFetchThroughAPI = url; 
  }

  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
    : url;
  const result = nodeFetch(urlAdaptedForScraping, newOptions);
  const timeoutId = setTimeout(() => controller.abort(), 20000);
  return result;
};

module.exports = fetch;