const Promise = require('bluebird'); const got = require('got'); const tunnel = require('tunnel'); const axios = require('axios').default; const HttpsProxyAgent = require('https-proxy-agent'); const SocksProxyAgent = require('socks-proxy-agent'); const { timeoutPromise } = require('./helpers'); // setup axios interceptors required to measure request times and timeout axios.interceptors.request.use( x => { // to avoid overwriting if another interceptor // already defined the same object (meta) x.meta = x.meta || {} x.meta.requestStartedAt = new Date().getTime(); return x; }); axios.interceptors.response.use(x => { x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt; return x; }); // *** end of axios setup const loadAllProxyServers = async () => { return { 'https': await loadProxyServersOfType('https'), 'socks5': await loadProxyServersOfType('socks5') } } const loadProxyServersOfType = async (proxyType) => { try { const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 }); const serversList = response.body.split('\r\n'); return serversList.map(proxyServer => { return { address: proxyServer, type: proxyType, timeToFetch: undefined } }); }catch(e){ console.log('[ERROR](loadProxyServersOfType)', e); return []; } } const setTimeToFetch = async (proxyList) => { const urlToFetch = process.env.URL_TO_FETCH.trim(); const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000; return await Promise.map(proxyList, async ({ address, type }, i) => { try{ if (address.length === 0) { throw new Error('Malformed Proxy URL'); } const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`); let responsePromise; if (type === 'https'){ // when using HTTPS, set 'httpAgent' instead of 'httpsAgent' responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout }); }else{ responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout }); } // setting timeout in axios object is not enough for some requests const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]); return { address, type, timeToFetch: responseObject.responseTime } }catch (e) { // console.log('[ERROR](setTimeToFetch)', type, address); return { address, type, timeToFetch: undefined } } }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5}); } exports.loadAllProxyServers = loadAllProxyServers; exports.setTimeToFetch = setTimeToFetch;