Add timeout to fetch wrapper

This commit is contained in:
=
2020-09-15 01:27:20 -07:00
parent ade3eb307d
commit 698acb010a
4 changed files with 28 additions and 5 deletions

View File

@@ -1,4 +1,5 @@
const nodeFetch = require("node-fetch");
const AbortController = require('abort-controller');
const {
USER_AGENT,
USE_SCRAPER_API,
@@ -11,10 +12,15 @@ const timeout = (ms) => {
}
const fetch = async (url, options = {}) => {
const controller = new AbortController();
const newOptions = Object.assign({}, options);
if (!newOptions["headers"]) {
newOptions["headers"] = {};
}
newOptions.signal = controller.signal;
// newOptions["headers"]["User-Agent"] = USER_AGENT;
let urlToFetchThroughAPI = Buffer.from(url).toString('base64');
@@ -25,7 +31,9 @@ const fetch = async (url, options = {}) => {
const urlAdaptedForScraping = USE_SCRAPER_API
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
: url;
return nodeFetch(urlAdaptedForScraping, newOptions);
const result = nodeFetch(urlAdaptedForScraping, newOptions);
const timeoutId = setTimeout(() => controller.abort(), 5000);
return result;
};
module.exports = fetch;