implement proxy speed check

proxyHelpers.js

@@ -0,0 +1,93 @@
+const Promise = require('bluebird');
+const got = require('got');
+const tunnel = require('tunnel');
+const axios = require('axios').default;
+const HttpsProxyAgent = require('https-proxy-agent');
+const SocksProxyAgent = require('socks-proxy-agent');
+
+// setup axios interceptors required to measure request times
+
+axios.interceptors.request.use( x => {
+    // to avoid overwriting if another interceptor
+    // already defined the same object (meta)
+    x.meta = x.meta || {}
+    x.meta.requestStartedAt = new Date().getTime();
+    return x;
+});
+
+axios.interceptors.response.use(x => {
+    x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
+    return x;
+});
+
+// *** end of axios setup
+
+
+const loadAllProxyServers = async () => {
+    return {
+        'https': await loadProxyServersOfType('https'),
+        'socks5': await loadProxyServersOfType('socks5')
+    }
+}
+
+const loadProxyServersOfType = async (proxyType) => {
+    try {
+        const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
+        const serversList = response.body.split('\r\n');
+        return serversList.map(proxyServer => {
+            return {
+                address: proxyServer,
+                type: proxyType,
+                timeToFetch: undefined
+            }
+        });
+    }catch(e){
+        console.log('[ERROR](loadProxyServersOfType)', e);
+        return [];
+    }
+}
+
+const setTimeToFetch = async (proxyList) => {
+    const urlToFetch = process.env.URL_TO_FETCH.trim();
+    return await Promise.map(proxyList, async ({ address, type }, i) => {
+        console.log('Analyzing ', type, i, '/', proxyList.length);
+        try{
+            if (address.length === 0) {
+                throw new Error('Malformed Proxy URL');
+            }
+            const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
+            let responseObject = null;
+            if (type === 'https'){
+                // when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
+              responseObject = await axios({ httpAgent: proxyAgent, url: urlToFetch });
+            }else{
+              responseObject = await axios({ httpsAgent: proxyAgent, url: urlToFetch });
+            }
+            return {
+                address,
+                type,
+                timeToFetch: responseObject.responseTime
+            }
+        }catch (e) {
+            console.log('[ERROR](setTimeToFetch)', type, address, e);
+            return {
+                address,
+                type,
+                timeToFetch: undefined
+            }
+        }
+    }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
+}
+
+const sortProxyServers = (proxyList) => {
+    return proxyList;
+}
+
+const selectBestProxies = (proxyList) => {
+    return proxyList;
+}
+
+exports.loadAllProxyServers = loadAllProxyServers;
+exports.setTimeToFetch = setTimeToFetch;
+exports.sortProxyServers = sortProxyServers;
+exports.selectBestProxies = selectBestProxies;