Add caching to fetch wrapper

This commit is contained in:
Senad Uka
2020-10-21 06:16:35 +02:00
parent 88f9d10586
commit 92e4f4ed5a
7 changed files with 38 additions and 6 deletions

View File

@@ -1,5 +1,9 @@
const nodeFetch = require("node-fetch");
const AbortController = require('abort-controller');
const FetchCache = require('@sozialhelden/fetch-cache').default;
console.log("Fc ", FetchCache)
const {
USER_AGENT,
USE_SCRAPER_API,
@@ -12,7 +16,20 @@ const timeout = (ms) => {
return new Promise(resolve => setTimeout(resolve, ms));
}
const fetch = async (url, options = {}) => {
const fetchCache = new FetchCache({
fetch: nodeFetch,
cacheOptions: {
// Don't save more than 100 responses in the cache. Allows infinite responses by default
maximalItemCount: 10000,
// When should the cache evict responses when its full?
evictExceedingItemsBy: 'age', // Valid values: 'lru' or 'age'
defaultTTL: 10 * 60 * 1000 // 10 minutes
// ...see https://github.com/sozialhelden/hamster-cache for all possible options
},
});
const fetch = async (url, options = {}, useCache = true) => {
const controller = new AbortController();
const newOptions = Object.assign({}, options);
@@ -32,7 +49,8 @@ const fetch = async (url, options = {}) => {
const urlAdaptedForScraping = USE_SCRAPER_API
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
: url;
const result = nodeFetch(urlAdaptedForScraping, newOptions);
const result = useCache ? fetchCache.fetch(urlAdaptedForScraping, newOptions) : nodeFetch(urlAdaptedForScraping, newOptions);
const timeoutId = setTimeout(() => controller.abort(), NODE_FETCH_TIMEOUT_MS);
return result;
};