const nodeFetch = require("node-fetch");
const {
  USER_AGENT,
  USE_SCRAPER_API,
  SCRAPER_API_KEY,
  SCRAPER_API_BASE_URL
} = require("../config/appConfig");

const async timeout = (method, ms) => {
    console.error("Waiting for ", ms);
    return new Promise(resolve => { 
      setTimeout(() => {
        await method();
        resolve();
      }, ms); 
    });
}

const fetch = async (url, options = {}) => {
  const newOptions = Object.assign({}, options);
  if (!newOptions["headers"]) {
    newOptions["headers"] = {};
  }
 // newOptions["headers"]["User-Agent"] = USER_AGENT;

  let urlToFetchThroughAPI = Buffer.from(url).toString('base64'); 
  if (SCRAPER_API_BASE_URL.includes('scraperapi')) { 
    urlToFetchThroughAPI = url; 
  }

  const urlAdaptedForScraping = USE_SCRAPER_API
    ? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
    : url;
  const randomPauseMS = Math.floor(Math.random() * Math.floor(1500)) + 500;
  return timeout(() => {  nodeFetch(urlAdaptedForScraping, newOptions); }, randomPauseMS); 
};

module.exports = fetch;