const nodeFetch = require("node-fetch");
const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY,
  SCRAPER_API_BASE_URL
} = require("../config/appConfig");

const fetch = async (url, options = {}) => {
  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }
  newOptions["headers"]["User-Agent"] = USER_AGENT;

  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${Buffer.from(url).toString('base64')}`
    : url;

  return nodeFetch(urlAdaptedForScraping, newOptions);
};

module.exports = fetch;