25 lines
632 B
JavaScript
25 lines
632 B
JavaScript
const nodeFetch = require("node-fetch");
|
|
const {
|
|
USER_AGENT,
|
|
USE_SCRAPER_API,
|
|
SCRAPER_API_KEY
|
|
} = require("../config/appConfig");
|
|
|
|
const fetch = async (url, options = {}) => {
|
|
const newOptions = Object.assign({}, options);
|
|
if (!newOptions["headers"]) {
|
|
newOptions["headers"] = {};
|
|
}
|
|
newOptions["headers"]["User-Agent"] = USER_AGENT;
|
|
const urlAdaptedForScraping = USE_SCRAPER_API
|
|
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
|
|
: url;
|
|
|
|
//
|
|
console.log("Url for scraping:", urlAdaptedForScraping);
|
|
|
|
return nodeFetch(urlAdaptedForScraping, newOptions);
|
|
};
|
|
|
|
module.exports = fetch;
|