implement proxy speed check

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

example.env

@@ -0,0 +1,8 @@
+SERVER_PORT=Port where server will listen for requests
+
+PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL
+PROXY_LIST_TIMEOUT = Wait PROXY_LIST_TIMEOUT seconds for proxy list before dropping request
+PROXY_LIST_RELOAD_INTERVAL = Refresh proxy list every PROXY_LIST_RELOAD_INTERVAL minutes
+
+MAX_PROXY_CONNECTIONS = Number of maximum parallel connections when testing proxy speed (Default 5)
+URL_TO_FETCH = Url to fetch for proxy speed evaluation

helpers.js

@@ -0,0 +1,14 @@
+const convertProxyListToString = (proxyList) => {
+    if (Array.isArray(proxyList)){
+        let result = '';
+        proxyList.forEach(proxyServer => {
+            result += `[${proxyServer.timeToFetch}]` + proxyServer.address + '\r\n';
+        });
+        return result;
+    }
+    console.log('[ERROR](convertProxyListToString) Proxy list is not an array')
+    console.log(proxyList);
+    return '';
+}
+
+exports.convertProxyListToString = convertProxyListToString;

index.js

@@ -0,0 +1,51 @@
+require('dotenv').config();
+const http = require('http');
+
+const { convertProxyListToString } = require('./helpers');
+const { loadAllProxyServers, setTimeToFetch, sortProxyServers, selectBestProxies } = require('./proxyHelpers');
+
+let proxyServersObject = {
+    'https': [],
+    'socks5': []
+};
+
+const handleHttpRequest = (req, res) => {
+    const proxyType = req.url.replace('/', '');
+    if (proxyType === "favicon.ico"){
+        res.end();
+        return;
+    }
+
+    if (proxyType === 'https' || proxyType === 'socks5'){
+        const responseToSend = convertProxyListToString(proxyServersObject[proxyType]);
+        res.write(responseToSend);
+    }else{
+        console.log('[WARN](handleHttpRequest) Proxy type is not valid (https/socks5) - got : ', proxyType);
+        res.write('');
+    }
+    res.end();
+}
+
+const refreshProxyList = async () => {
+    const fullProxyList = await loadAllProxyServers();
+    const httpsProxyList = fullProxyList['https'];
+    // const socks5ProxyList = fullProxyList['socks5'];
+
+    const updatedHttpsProxyList = await setTimeToFetch(httpsProxyList);
+    // const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList);
+
+    const sortedHttpsProxyList = sortProxyServers(updatedHttpsProxyList);
+    // const sortedSocks5ProxyList = sortProxyServers(updatedSocks5ProxyList);
+
+    proxyServersObject = {
+        'https': selectBestProxies(sortedHttpsProxyList),
+        // 'socks5': selectBestProxies(sortedSocks5ProxyList)
+    }
+}
+
+(async () => {
+    await refreshProxyList();
+})();
+http.createServer(handleHttpRequest).listen(process.env.SERVER_PORT);
+setInterval(refreshProxyList, parseInt(process.env.PROXY_LIST_RELOAD_INTERVAL)*60*1000);
+

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "kiviproxylist",
+  "version": "1.0.0",
+  "description": "Kivi Proxy List maintaining app",
+  "main": "index.js",
+  "license": "MIT",
+  "scripts": {
+    "start": "node ./index.js",
+    "start-live": "nodemon ./index.js"
+  },
+  "dependencies": {
+    "axios": "^0.19.2",
+    "bluebird": "^3.7.2",
+    "dotenv": "^8.2.0",
+    "got": "^11.5.2",
+    "http": "^0.0.1-security",
+    "https-proxy-agent": "^5.0.0",
+    "querystring": "^0.2.0",
+    "socks-proxy-agent": "^5.0.0",
+    "tunnel": "^0.0.6"
+  },
+  "devDependencies": {
+    "nodemon": "^2.0.4"
+  }
+}

proxyHelpers.js

@@ -0,0 +1,93 @@
+const Promise = require('bluebird');
+const got = require('got');
+const tunnel = require('tunnel');
+const axios = require('axios').default;
+const HttpsProxyAgent = require('https-proxy-agent');
+const SocksProxyAgent = require('socks-proxy-agent');
+
+// setup axios interceptors required to measure request times
+
+axios.interceptors.request.use( x => {
+    // to avoid overwriting if another interceptor
+    // already defined the same object (meta)
+    x.meta = x.meta || {}
+    x.meta.requestStartedAt = new Date().getTime();
+    return x;
+});
+
+axios.interceptors.response.use(x => {
+    x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
+    return x;
+});
+
+// *** end of axios setup
+
+
+const loadAllProxyServers = async () => {
+    return {
+        'https': await loadProxyServersOfType('https'),
+        'socks5': await loadProxyServersOfType('socks5')
+    }
+}
+
+const loadProxyServersOfType = async (proxyType) => {
+    try {
+        const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
+        const serversList = response.body.split('\r\n');
+        return serversList.map(proxyServer => {
+            return {
+                address: proxyServer,
+                type: proxyType,
+                timeToFetch: undefined
+            }
+        });
+    }catch(e){
+        console.log('[ERROR](loadProxyServersOfType)', e);
+        return [];
+    }
+}
+
+const setTimeToFetch = async (proxyList) => {
+    const urlToFetch = process.env.URL_TO_FETCH.trim();
+    return await Promise.map(proxyList, async ({ address, type }, i) => {
+        console.log('Analyzing ', type, i, '/', proxyList.length);
+        try{
+            if (address.length === 0) {
+                throw new Error('Malformed Proxy URL');
+            }
+            const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
+            let responseObject = null;
+            if (type === 'https'){
+                // when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
+              responseObject = await axios({ httpAgent: proxyAgent, url: urlToFetch });
+            }else{
+              responseObject = await axios({ httpsAgent: proxyAgent, url: urlToFetch });
+            }
+            return {
+                address,
+                type,
+                timeToFetch: responseObject.responseTime
+            }
+        }catch (e) {
+            console.log('[ERROR](setTimeToFetch)', type, address, e);
+            return {
+                address,
+                type,
+                timeToFetch: undefined
+            }
+        }
+    }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
+}
+
+const sortProxyServers = (proxyList) => {
+    return proxyList;
+}
+
+const selectBestProxies = (proxyList) => {
+    return proxyList;
+}
+
+exports.loadAllProxyServers = loadAllProxyServers;
+exports.setTimeToFetch = setTimeToFetch;
+exports.sortProxyServers = sortProxyServers;
+exports.selectBestProxies = selectBestProxies;

test.js

@@ -0,0 +1,29 @@
+const axios = require('axios').default
+const SocksProxyAgent = require('socks-proxy-agent');
+const HttpsProxyAgent = require('https-proxy-agent');
+
+const socksProxyOptions = `socks5://114.103.105.105:4216`; // your sock5 host and port;
+const httpsProxyOptions = 'https://92.119.222.1:8080';
+
+const socksAgent = new SocksProxyAgent(socksProxyOptions);
+const httpAgent = new HttpsProxyAgent(httpsProxyOptions);
+
+axios.interceptors.request.use( x => {
+    // to avoid overwriting if another interceptor
+    // already defined the same object (meta)
+    x.meta = x.meta || {}
+    x.meta.requestStartedAt = new Date().getTime();
+    return x;
+});
+
+axios.interceptors.response.use(x => {
+    x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
+    return x;
+});
+
+axios({
+    httpsAgent: socksAgent,
+    url:'https://api.ipify.org/?format=json'
+}).then(r => {
+    console.log(r.responseTime);
+})