From 7adcbe31022e7a618192d16722693783c5f7839a Mon Sep 17 00:00:00 2001 From: Bilal Date: Thu, 20 Aug 2020 15:54:39 +0300 Subject: [PATCH] implement socks5 proxy measurement --- example.env | 3 ++- helpers.js | 31 +++++++++++++++++++++++++++++-- index.js | 19 ++++++++++++------- proxyHelpers.js | 31 ++++++++++++++----------------- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/example.env b/example.env index b26834d..9d70785 100644 --- a/example.env +++ b/example.env @@ -5,4 +5,5 @@ PROXY_LIST_TIMEOUT = Wait PROXY_LIST_TIMEOUT seconds for proxy list before dropp PROXY_LIST_RELOAD_INTERVAL = Refresh proxy list every PROXY_LIST_RELOAD_INTERVAL minutes MAX_PROXY_CONNECTIONS = Number of maximum parallel connections when testing proxy speed (Default 5) -URL_TO_FETCH = Url to fetch for proxy speed evaluation \ No newline at end of file +URL_TO_FETCH = Url to fetch for proxy speed evaluation +SINGLE_PROXY_TIMEOUT = Timeout in seconds when testing single proxy; if proxy do not respond in this time, it will be removed from list (Default is 30 seconds) \ No newline at end of file diff --git a/helpers.js b/helpers.js index 7a23bf1..cfde2c2 100644 --- a/helpers.js +++ b/helpers.js @@ -1,8 +1,24 @@ +const sortProxyServers = (proxyList) => { + return proxyList.sort((proxyServer1, proxyServer2) => { + if (proxyServer1 && proxyServer1.timeToFetch && proxyServer2 && proxyServer2.timeToFetch) { + return proxyServer1.timeToFetch > proxyServer2.timeToFetch ? 1 : -1; + } + }); +} + +const selectBestProxies = (proxyList) => { + return proxyList; +} + +const cleanProxyServers = (proxyList) => { + return proxyList.filter(proxyServer => proxyServer && proxyServer.timeToFetch); +} + const convertProxyListToString = (proxyList) => { if (Array.isArray(proxyList)){ let result = ''; proxyList.forEach(proxyServer => { - result += `[${proxyServer.timeToFetch}]` + proxyServer.address + '\r\n'; + result += `${proxyServer.address}\r\n`; }); return result; } @@ -11,4 +27,15 @@ const convertProxyListToString = (proxyList) => { return ''; } -exports.convertProxyListToString = convertProxyListToString; \ No newline at end of file +const timeoutPromise = (timeout) => { + return new Promise((resolve, reject) => { + setTimeout(() => { reject('Timeout') }, timeout); + }); +} + + +exports.sortProxyServers = sortProxyServers; +exports.selectBestProxies = selectBestProxies; +exports.cleanProxyServers = cleanProxyServers; +exports.convertProxyListToString = convertProxyListToString; +exports.timeoutPromise = timeoutPromise; diff --git a/index.js b/index.js index 049675b..aa46e8b 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,8 @@ require('dotenv').config(); const http = require('http'); -const { convertProxyListToString } = require('./helpers'); -const { loadAllProxyServers, setTimeToFetch, sortProxyServers, selectBestProxies } = require('./proxyHelpers'); +const { loadAllProxyServers, setTimeToFetch } = require('./proxyHelpers'); +const { sortProxyServers, selectBestProxies, cleanProxyServers, convertProxyListToString } = require('./helpers'); let proxyServersObject = { 'https': [], @@ -27,20 +27,25 @@ const handleHttpRequest = (req, res) => { } const refreshProxyList = async () => { + console.log(new Date(), 'Refreshing proxy list--------'); const fullProxyList = await loadAllProxyServers(); const httpsProxyList = fullProxyList['https']; - // const socks5ProxyList = fullProxyList['socks5']; + const socks5ProxyList = fullProxyList['socks5']; const updatedHttpsProxyList = await setTimeToFetch(httpsProxyList); - // const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList); + const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList); - const sortedHttpsProxyList = sortProxyServers(updatedHttpsProxyList); - // const sortedSocks5ProxyList = sortProxyServers(updatedSocks5ProxyList); + const cleanUpdatedHttpsProxyList = cleanProxyServers(updatedHttpsProxyList); + const cleanUpdatedSocks5ProxyList = cleanProxyServers(updatedSocks5ProxyList); + + const sortedHttpsProxyList = sortProxyServers(cleanUpdatedHttpsProxyList); + const sortedSocks5ProxyList = sortProxyServers(cleanUpdatedSocks5ProxyList); proxyServersObject = { 'https': selectBestProxies(sortedHttpsProxyList), - // 'socks5': selectBestProxies(sortedSocks5ProxyList) + 'socks5': selectBestProxies(sortedSocks5ProxyList) } + console.log(new Date(), 'DONE----------------------', (process.memoryUsage()['rss'] / 1024 * 100) / 100, 'KiB'); } (async () => { diff --git a/proxyHelpers.js b/proxyHelpers.js index ec56724..1a47ea9 100644 --- a/proxyHelpers.js +++ b/proxyHelpers.js @@ -5,7 +5,9 @@ const axios = require('axios').default; const HttpsProxyAgent = require('https-proxy-agent'); const SocksProxyAgent = require('socks-proxy-agent'); -// setup axios interceptors required to measure request times +const { timeoutPromise } = require('./helpers'); + +// setup axios interceptors required to measure request times and timeout axios.interceptors.request.use( x => { // to avoid overwriting if another interceptor @@ -49,27 +51,32 @@ const loadProxyServersOfType = async (proxyType) => { const setTimeToFetch = async (proxyList) => { const urlToFetch = process.env.URL_TO_FETCH.trim(); + const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000; + return await Promise.map(proxyList, async ({ address, type }, i) => { - console.log('Analyzing ', type, i, '/', proxyList.length); try{ if (address.length === 0) { throw new Error('Malformed Proxy URL'); } const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`); - let responseObject = null; + let responsePromise; + if (type === 'https'){ // when using HTTPS, set 'httpAgent' instead of 'httpsAgent' - responseObject = await axios({ httpAgent: proxyAgent, url: urlToFetch }); + responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout }); }else{ - responseObject = await axios({ httpsAgent: proxyAgent, url: urlToFetch }); + responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout }); } + + // setting timeout in axios object is not enough for some requests + const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]); return { address, type, timeToFetch: responseObject.responseTime } }catch (e) { - console.log('[ERROR](setTimeToFetch)', type, address, e); + console.log('[ERROR](setTimeToFetch)', type, address); return { address, type, @@ -79,15 +86,5 @@ const setTimeToFetch = async (proxyList) => { }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5}); } -const sortProxyServers = (proxyList) => { - return proxyList; -} - -const selectBestProxies = (proxyList) => { - return proxyList; -} - exports.loadAllProxyServers = loadAllProxyServers; -exports.setTimeToFetch = setTimeToFetch; -exports.sortProxyServers = sortProxyServers; -exports.selectBestProxies = selectBestProxies; \ No newline at end of file +exports.setTimeToFetch = setTimeToFetch; \ No newline at end of file