old-web/app/helpers/fetchWrapper.js

const nodeFetch = require("node-fetch");
const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY
} = require("../config/appConfig");

const fetch = async (url, options = {}) => {
  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }
  newOptions["headers"]["User-Agent"] = USER_AGENT;
  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
    : url;

  //
  // console.log("Url for scraping:", urlAdaptedForScraping);

  return nodeFetch(urlAdaptedForScraping, newOptions);
};

module.exports = fetch;