old-proxylist/proxyHelpers.js

const Promise = require('bluebird');
const got = require('got');
const tunnel = require('tunnel');
const axios = require('axios').default;
const HttpsProxyAgent = require('https-proxy-agent');
const SocksProxyAgent = require('socks-proxy-agent');

const { timeoutPromise } = require('./helpers');

// setup axios interceptors required to measure request times and timeout

axios.interceptors.request.use( x => {
    // to avoid overwriting if another interceptor
    // already defined the same object (meta)
    x.meta = x.meta || {}
    x.meta.requestStartedAt = new Date().getTime();
    return x;
});

axios.interceptors.response.use(x => {
    x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
    return x;
});

// *** end of axios setup


const loadAllProxyServers = async () => {
    return {
        'https': await loadProxyServersOfType('https'),
        'socks5': await loadProxyServersOfType('socks5')
    }
}

const loadProxyServersOfType = async (proxyType) => {
    try {
        const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
        const serversList = response.body.split('\r\n');
        return serversList.map(proxyServer => {
            return {
                address: proxyServer,
                type: proxyType,
                timeToFetch: undefined
            }
        });
    }catch(e){
        console.log('[ERROR](loadProxyServersOfType)', e);
        return [];
    }
}

const setTimeToFetch = async (proxyList) => {
    const urlToFetch = process.env.URL_TO_FETCH.trim();
    const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000;

    return await Promise.map(proxyList, async ({ address, type }, i) => {
        try{
            if (address.length === 0) {
                throw new Error('Malformed Proxy URL');
            }
            const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
            let responsePromise;

            if (type === 'https'){
                // when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
              responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout });
            }else{
              responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout });
            }

            // setting timeout in axios object is not enough for some requests
            const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]);
            return {
                address,
                type,
                timeToFetch: responseObject.responseTime
            }
        }catch (e) {
            // console.log('[ERROR](setTimeToFetch)', type, address);
            return {
                address,
                type,
                timeToFetch: undefined
            }
        }
    }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
}

exports.loadAllProxyServers = loadAllProxyServers;
exports.setTimeToFetch = setTimeToFetch;