Added Scraper API option.
This commit is contained in:
@@ -1,5 +1,9 @@
|
||||
const nodeFetch = require("node-fetch");
|
||||
const { USER_AGENT } = require("../config/appConfig");
|
||||
const {
|
||||
USER_AGENT,
|
||||
USE_SCRAPER_API,
|
||||
SCRAPER_API_KEY
|
||||
} = require("../config/appConfig");
|
||||
|
||||
const fetch = async (url, options = {}) => {
|
||||
const newOptions = Object.assign({}, options);
|
||||
@@ -7,7 +11,11 @@ const fetch = async (url, options = {}) => {
|
||||
newOptions["headers"] = {};
|
||||
}
|
||||
newOptions["headers"]["User-Agent"] = USER_AGENT;
|
||||
return nodeFetch(url, newOptions);
|
||||
const newUrl = USE_SCRAPER_API
|
||||
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
|
||||
: url;
|
||||
|
||||
return nodeFetch(newUrl, newOptions);
|
||||
};
|
||||
|
||||
module.exports = fetch;
|
||||
|
||||
Reference in New Issue
Block a user