59 lines
1.7 KiB
JavaScript
59 lines
1.7 KiB
JavaScript
const nodeFetch = require("node-fetch");
|
|
const AbortController = require('abort-controller');
|
|
const FetchCache = require('@sozialhelden/fetch-cache').default;
|
|
|
|
console.log("Fc ", FetchCache)
|
|
|
|
const {
|
|
USER_AGENT,
|
|
USE_SCRAPER_API,
|
|
SCRAPER_API_KEY,
|
|
SCRAPER_API_BASE_URL,
|
|
NODE_FETCH_TIMEOUT_MS
|
|
} = require("../config/appConfig");
|
|
|
|
const timeout = (ms) => {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
const fetchCache = new FetchCache({
|
|
fetch: nodeFetch,
|
|
cacheOptions: {
|
|
// Don't save more than 100 responses in the cache. Allows infinite responses by default
|
|
maximalItemCount: 10000,
|
|
// When should the cache evict responses when its full?
|
|
evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
|
|
defaultTTL: 6 * 60 * 60 * 1000 // 6 hours
|
|
// ...see https://github.com/sozialhelden/hamster-cache for all possible options
|
|
},
|
|
});
|
|
|
|
|
|
const fetch = async (url, options = {}, useCache = true) => {
|
|
const controller = new AbortController();
|
|
|
|
const newOptions = Object.assign({}, options);
|
|
if (!newOptions["headers"]) {
|
|
newOptions["headers"] = {};
|
|
}
|
|
|
|
newOptions.signal = controller.signal;
|
|
|
|
// newOptions["headers"]["User-Agent"] = USER_AGENT;
|
|
|
|
let urlToFetchThroughAPI = Buffer.from(url).toString('base64');
|
|
if (SCRAPER_API_BASE_URL.includes('scraperapi')) {
|
|
urlToFetchThroughAPI = url;
|
|
}
|
|
|
|
const urlAdaptedForScraping = USE_SCRAPER_API
|
|
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
|
|
: url;
|
|
const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);
|
|
|
|
const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
|
|
return result;
|
|
};
|
|
|
|
module.exports = fetch;
|