old-web/app/helpers/fetchWrapper.js

const nodeFetch = require("node-fetch");
const AbortController = require('abort-controller');
const FetchCache = require('@sozialhelden/fetch-cache').default;

console.log("Fc ", FetchCache)

const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY,
  SCRAPER_API_BASE_URL,
  NODE_FETCH_TIMEOUT_MS
} = require("../config/appConfig");

const timeout = (ms) => {
    return new Promise(resolve => setTimeout(resolve, ms));
}

const fetchCache = new FetchCache({
  fetch: nodeFetch,
  cacheOptions: {
    // Don't save more than 100 responses in the cache. Allows infinite responses by default
    maximalItemCount: 10000,
    // When should the cache evict responses when its full?
    evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
    defaultTTL: 6 * 60 * 60 * 1000  // 6 hours
    // ...see https://github.com/sozialhelden/hamster-cache for all possible options
  },
});


const fetch = async (url, options = {}, useCache = true) => {
  const controller = new AbortController();

  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }

  newOptions.signal = controller.signal;

 // newOptions["headers"]["User-Agent"] = USER_AGENT;

  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
    urlToFetchThroughAPI = url; 
  }

  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
    : url;
  const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);

  const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
  return result;
};

module.exports = fetch;