Add timeout to fetch wrapper
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
const nodeFetch = require("node-fetch");
|
||||
const AbortController = require('abort-controller');
|
||||
const {
|
||||
USER_AGENT,
|
||||
USE_SCRAPER_API,
|
||||
@@ -11,10 +12,15 @@ const timeout = (ms) => {
|
||||
}
|
||||
|
||||
const fetch = async (url, options = {}) => {
|
||||
const controller = new AbortController();
|
||||
|
||||
const newOptions = Object.assign({}, options);
|
||||
if (!newOptions["headers"]) {
|
||||
newOptions["headers"] = {};
|
||||
}
|
||||
|
||||
newOptions.signal = controller.signal;
|
||||
|
||||
// newOptions["headers"]["User-Agent"] = USER_AGENT;
|
||||
|
||||
let urlToFetchThroughAPI = Buffer.from(url).toString('base64');
|
||||
@@ -25,7 +31,9 @@ const fetch = async (url, options = {}) => {
|
||||
const urlAdaptedForScraping = USE_SCRAPER_API
|
||||
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
|
||||
: url;
|
||||
return nodeFetch(urlAdaptedForScraping, newOptions);
|
||||
const result = nodeFetch(urlAdaptedForScraping, newOptions);
|
||||
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
||||
return result;
|
||||
};
|
||||
|
||||
module.exports = fetch;
|
||||
|
||||
Reference in New Issue
Block a user