Added Scraper API option.

This commit is contained in:
Naida Vatric
2020-02-22 22:15:27 +01:00
parent 6791a509d0
commit d436d4a37b
3 changed files with 20 additions and 3 deletions

View File

@@ -1,5 +1,9 @@
const nodeFetch = require("node-fetch");
const { USER_AGENT } = require("../config/appConfig");
const {
USER_AGENT,
USE_SCRAPER_API,
SCRAPER_API_KEY
} = require("../config/appConfig");
const fetch = async (url, options = {}) => {
const newOptions = Object.assign({}, options);
@@ -7,7 +11,11 @@ const fetch = async (url, options = {}) => {
newOptions["headers"] = {};
}
newOptions["headers"]["User-Agent"] = USER_AGENT;
return nodeFetch(url, newOptions);
const newUrl = USE_SCRAPER_API
? `http://api.scraperapi.com/?api_key=${SCRAPER_API_KEY}&url=${url}`
: url;
return nodeFetch(newUrl, newOptions);
};
module.exports = fetch;