const nodeFetch = require("node-fetch");
const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY,
  SCRAPER_API_BASE_URL
} = require("../config/appConfig");

const fetch = async (url, options = {}) => {
  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }
 // newOptions["headers"]["User-Agent"] = USER_AGENT;

  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
    urlToFetchThroughAPI = url; 
  }

  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
    : url;

  return nodeFetch(urlAdaptedForScraping, newOptions);
};

module.exports = fetch;