39 lines
1.1 KiB
JavaScript
39 lines
1.1 KiB
JavaScript
const nodeFetch = require("node-fetch");
|
|
const {
|
|
USER_AGENT,
|
|
USE_SCRAPER_API,
|
|
SCRAPER_API_KEY,
|
|
SCRAPER_API_BASE_URL
|
|
} = require("../config/appConfig");
|
|
|
|
const async timeout = (method, ms) => {
|
|
console.error("Waiting for ", ms);
|
|
return new Promise(resolve => {
|
|
setTimeout(() => {
|
|
await method();
|
|
resolve();
|
|
}, ms);
|
|
});
|
|
}
|
|
|
|
const fetch = async (url, options = {}) => {
|
|
const newOptions = Object.assign({}, options);
|
|
if (!newOptions["headers"]) {
|
|
newOptions["headers"] = {};
|
|
}
|
|
// newOptions["headers"]["User-Agent"] = USER_AGENT;
|
|
|
|
let urlToFetchThroughAPI = Buffer.from(url).toString('base64');
|
|
if (SCRAPER_API_BASE_URL.includes('scraperapi')) {
|
|
urlToFetchThroughAPI = url;
|
|
}
|
|
|
|
const urlAdaptedForScraping = USE_SCRAPER_API
|
|
? `${SCRAPER_API_BASE_URL}?api_key=${SCRAPER_API_KEY}&url=${urlToFetchThroughAPI}`
|
|
: url;
|
|
const randomPauseMS = Math.floor(Math.random() * Math.floor(1500)) + 500;
|
|
return timeout(() => { nodeFetch(urlAdaptedForScraping, newOptions); }, randomPauseMS);
|
|
};
|
|
|
|
module.exports = fetch;
|