implement socks5 proxy measurement

example.env

@@ -5,4 +5,5 @@ PROXY_LIST_TIMEOUT = Wait PROXY_LIST_TIMEOUT seconds for proxy list before dropp
 PROXY_LIST_RELOAD_INTERVAL = Refresh proxy list every PROXY_LIST_RELOAD_INTERVAL minutes
 
 MAX_PROXY_CONNECTIONS = Number of maximum parallel connections when testing proxy speed (Default 5)
-URL_TO_FETCH = Url to fetch for proxy speed evaluation
+URL_TO_FETCH = Url to fetch for proxy speed evaluation
+SINGLE_PROXY_TIMEOUT = Timeout in seconds when testing single proxy; if proxy do not respond in this time, it will be removed from list (Default is 30 seconds)

helpers.js

@@ -1,8 +1,24 @@
+const sortProxyServers = (proxyList) => {
+    return proxyList.sort((proxyServer1, proxyServer2) => {
+        if (proxyServer1 && proxyServer1.timeToFetch && proxyServer2 && proxyServer2.timeToFetch) {
+            return proxyServer1.timeToFetch > proxyServer2.timeToFetch ? 1 : -1;
+        }
+    });
+}
+
+const selectBestProxies = (proxyList) => {
+    return proxyList;
+}
+
+const cleanProxyServers = (proxyList) => {
+    return proxyList.filter(proxyServer => proxyServer && proxyServer.timeToFetch);
+}
+
 const convertProxyListToString = (proxyList) => {
     if (Array.isArray(proxyList)){
         let result = '';
         proxyList.forEach(proxyServer => {
-            result += `[${proxyServer.timeToFetch}]` + proxyServer.address + '\r\n';
+            result += `${proxyServer.address}\r\n`;
         });
         return result;
     }
@@ -11,4 +27,15 @@ const convertProxyListToString = (proxyList) => {
     return '';
 }
 
-exports.convertProxyListToString = convertProxyListToString;
+const timeoutPromise = (timeout) => {
+    return new Promise((resolve, reject) => {
+        setTimeout(() => { reject('Timeout') }, timeout);
+    });
+}
+
+
+exports.sortProxyServers = sortProxyServers;
+exports.selectBestProxies = selectBestProxies;
+exports.cleanProxyServers = cleanProxyServers;
+exports.convertProxyListToString = convertProxyListToString;
+exports.timeoutPromise = timeoutPromise;

index.js

@@ -1,8 +1,8 @@
 require('dotenv').config();
 const http = require('http');
 
-const { convertProxyListToString } = require('./helpers');
-const { loadAllProxyServers, setTimeToFetch, sortProxyServers, selectBestProxies } = require('./proxyHelpers');
+const { loadAllProxyServers, setTimeToFetch } = require('./proxyHelpers');
+const { sortProxyServers, selectBestProxies, cleanProxyServers, convertProxyListToString } = require('./helpers');
 
 let proxyServersObject = {
     'https': [],
@@ -27,20 +27,25 @@ const handleHttpRequest = (req, res) => {
 }
 
 const refreshProxyList = async () => {
+    console.log(new Date(), 'Refreshing proxy list--------');
     const fullProxyList = await loadAllProxyServers();
     const httpsProxyList = fullProxyList['https'];
-    // const socks5ProxyList = fullProxyList['socks5'];
+    const socks5ProxyList = fullProxyList['socks5'];
 
     const updatedHttpsProxyList = await setTimeToFetch(httpsProxyList);
-    // const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList);
+    const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList);
 
-    const sortedHttpsProxyList = sortProxyServers(updatedHttpsProxyList);
-    // const sortedSocks5ProxyList = sortProxyServers(updatedSocks5ProxyList);
+    const cleanUpdatedHttpsProxyList = cleanProxyServers(updatedHttpsProxyList);
+    const cleanUpdatedSocks5ProxyList = cleanProxyServers(updatedSocks5ProxyList);
+
+    const sortedHttpsProxyList = sortProxyServers(cleanUpdatedHttpsProxyList);
+    const sortedSocks5ProxyList = sortProxyServers(cleanUpdatedSocks5ProxyList);
 
     proxyServersObject = {
         'https': selectBestProxies(sortedHttpsProxyList),
-        // 'socks5': selectBestProxies(sortedSocks5ProxyList)
+        'socks5': selectBestProxies(sortedSocks5ProxyList)
     }
+    console.log(new Date(), 'DONE----------------------', (process.memoryUsage()['rss'] / 1024 * 100) / 100, 'KiB');
 }
 
 (async () => {

proxyHelpers.js

@@ -5,7 +5,9 @@ const axios = require('axios').default;
 const HttpsProxyAgent = require('https-proxy-agent');
 const SocksProxyAgent = require('socks-proxy-agent');
 
-// setup axios interceptors required to measure request times
+const { timeoutPromise } = require('./helpers');
+
+// setup axios interceptors required to measure request times and timeout
 
 axios.interceptors.request.use( x => {
     // to avoid overwriting if another interceptor
@@ -49,27 +51,32 @@ const loadProxyServersOfType = async (proxyType) => {
 
 const setTimeToFetch = async (proxyList) => {
     const urlToFetch = process.env.URL_TO_FETCH.trim();
+    const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000;
+
     return await Promise.map(proxyList, async ({ address, type }, i) => {
-        console.log('Analyzing ', type, i, '/', proxyList.length);
         try{
             if (address.length === 0) {
                 throw new Error('Malformed Proxy URL');
             }
             const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
-            let responseObject = null;
+            let responsePromise;
+
             if (type === 'https'){
                 // when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
-              responseObject = await axios({ httpAgent: proxyAgent, url: urlToFetch });
+              responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout });
             }else{
-              responseObject = await axios({ httpsAgent: proxyAgent, url: urlToFetch });
+              responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout });
             }
+
+            // setting timeout in axios object is not enough for some requests
+            const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]);
             return {
                 address,
                 type,
                 timeToFetch: responseObject.responseTime
             }
         }catch (e) {
-            console.log('[ERROR](setTimeToFetch)', type, address, e);
+            console.log('[ERROR](setTimeToFetch)', type, address);
             return {
                 address,
                 type,
@@ -79,15 +86,5 @@ const setTimeToFetch = async (proxyList) => {
     }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
 }
 
-const sortProxyServers = (proxyList) => {
-    return proxyList;
-}
-
-const selectBestProxies = (proxyList) => {
-    return proxyList;
-}
-
 exports.loadAllProxyServers = loadAllProxyServers;
-exports.setTimeToFetch = setTimeToFetch;
-exports.sortProxyServers = sortProxyServers;
-exports.selectBestProxies = selectBestProxies;
+exports.setTimeToFetch = setTimeToFetch;