.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

README.md

@@ -1,2 +1,16 @@
 # proxylist
 
+### Requirements
+Node version : `v13.13.0`  
+Yarn version : `1.22.4`
+
+### Setup
+
+1. Copy `example.env` to the `.env` and set values as desired
+2. Execute `yarn install`
+
+Run server with `yarn start`
+
+### Development
+
+Server can be started with `yarn run start-live` for live reload

example.env

@@ -0,0 +1,9 @@
+SERVER_PORT=Port where server will listen for requests
+
+PROXY_LIST_BASE_URL = Base url where proxy list can be fetched. Proxy type(https, socks5) will be concatenated to this URL
+PROXY_LIST_TIMEOUT = Wait PROXY_LIST_TIMEOUT seconds for proxy list before dropping request
+PROXY_LIST_RELOAD_INTERVAL = Refresh proxy list every PROXY_LIST_RELOAD_INTERVAL minutes
+
+MAX_PROXY_CONNECTIONS = Number of maximum parallel connections when testing proxy speed (Default 5)
+URL_TO_FETCH = Url to fetch for proxy speed evaluation
+SINGLE_PROXY_TIMEOUT = Timeout in seconds when testing single proxy; if proxy do not respond in this time, it will be removed from list (Default is 30 seconds)

helpers.js

@@ -0,0 +1,41 @@
+const sortProxyServers = (proxyList) => {
+    return proxyList.sort((proxyServer1, proxyServer2) => {
+        if (proxyServer1 && proxyServer1.timeToFetch && proxyServer2 && proxyServer2.timeToFetch) {
+            return proxyServer1.timeToFetch > proxyServer2.timeToFetch ? 1 : -1;
+        }
+    });
+}
+
+const selectBestProxies = (proxyList) => {
+    return proxyList;
+}
+
+const cleanProxyServers = (proxyList) => {
+    return proxyList.filter(proxyServer => proxyServer && proxyServer.timeToFetch);
+}
+
+const convertProxyListToString = (proxyList) => {
+    if (Array.isArray(proxyList)){
+        let result = '';
+        proxyList.forEach(proxyServer => {
+            result += `${proxyServer.address}\r\n`;
+        });
+        return result;
+    }
+    console.log('[ERROR](convertProxyListToString) Proxy list is not an array')
+    console.log(proxyList);
+    return '';
+}
+
+const timeoutPromise = (timeout) => {
+    return new Promise((resolve, reject) => {
+        setTimeout(() => { reject('Timeout') }, timeout);
+    });
+}
+
+
+exports.sortProxyServers = sortProxyServers;
+exports.selectBestProxies = selectBestProxies;
+exports.cleanProxyServers = cleanProxyServers;
+exports.convertProxyListToString = convertProxyListToString;
+exports.timeoutPromise = timeoutPromise;

index.js

@@ -0,0 +1,56 @@
+require('dotenv').config();
+const http = require('http');
+
+const { loadAllProxyServers, setTimeToFetch } = require('./proxyHelpers');
+const { sortProxyServers, selectBestProxies, cleanProxyServers, convertProxyListToString } = require('./helpers');
+
+let proxyServersObject = {
+    'https': [],
+    'socks5': []
+};
+
+const handleHttpRequest = (req, res) => {
+    const proxyType = req.url.replace('/', '');
+    if (proxyType === "favicon.ico"){
+        res.end();
+        return;
+    }
+
+    if (proxyType === 'https' || proxyType === 'socks5'){
+        const responseToSend = convertProxyListToString(proxyServersObject[proxyType]);
+        res.write(responseToSend);
+    }else{
+        console.log('[WARN](handleHttpRequest) Proxy type is not valid (https/socks5) - got : ', proxyType);
+        res.write('');
+    }
+    res.end();
+}
+
+const refreshProxyList = async () => {
+    console.log(new Date(), 'Refreshing proxy list--------');
+    const fullProxyList = await loadAllProxyServers();
+    const httpsProxyList = fullProxyList['https'];
+    const socks5ProxyList = fullProxyList['socks5'];
+
+    const updatedHttpsProxyList = await setTimeToFetch(httpsProxyList);
+    const updatedSocks5ProxyList = await setTimeToFetch(socks5ProxyList);
+
+    const cleanUpdatedHttpsProxyList = cleanProxyServers(updatedHttpsProxyList);
+    const cleanUpdatedSocks5ProxyList = cleanProxyServers(updatedSocks5ProxyList);
+
+    const sortedHttpsProxyList = sortProxyServers(cleanUpdatedHttpsProxyList);
+    const sortedSocks5ProxyList = sortProxyServers(cleanUpdatedSocks5ProxyList);
+
+    proxyServersObject = {
+        'https': selectBestProxies(sortedHttpsProxyList),
+        'socks5': selectBestProxies(sortedSocks5ProxyList)
+    }
+    console.log(new Date(), 'DONE----------------------', (process.memoryUsage()['rss'] / 1024 * 100) / 100, 'KiB');
+}
+
+(async () => {
+    await refreshProxyList();
+})();
+http.createServer(handleHttpRequest).listen(process.env.SERVER_PORT);
+setInterval(refreshProxyList, parseInt(process.env.PROXY_LIST_RELOAD_INTERVAL)*60*1000);
+

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "kiviproxylist",
+  "version": "1.0.0",
+  "description": "Kivi Proxy List maintaining app",
+  "main": "index.js",
+  "license": "MIT",
+  "scripts": {
+    "start": "node ./index.js",
+    "start-live": "nodemon ./index.js"
+  },
+  "dependencies": {
+    "axios": "^0.19.2",
+    "bluebird": "^3.7.2",
+    "dotenv": "^8.2.0",
+    "got": "^11.5.2",
+    "http": "^0.0.1-security",
+    "https-proxy-agent": "^5.0.0",
+    "querystring": "^0.2.0",
+    "socks-proxy-agent": "^5.0.0",
+    "tunnel": "^0.0.6"
+  },
+  "devDependencies": {
+    "nodemon": "^2.0.4"
+  }
+}

proxyHelpers.js

@@ -0,0 +1,90 @@
+const Promise = require('bluebird');
+const got = require('got');
+const tunnel = require('tunnel');
+const axios = require('axios').default;
+const HttpsProxyAgent = require('https-proxy-agent');
+const SocksProxyAgent = require('socks-proxy-agent');
+
+const { timeoutPromise } = require('./helpers');
+
+// setup axios interceptors required to measure request times and timeout
+
+axios.interceptors.request.use( x => {
+    // to avoid overwriting if another interceptor
+    // already defined the same object (meta)
+    x.meta = x.meta || {}
+    x.meta.requestStartedAt = new Date().getTime();
+    return x;
+});
+
+axios.interceptors.response.use(x => {
+    x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
+    return x;
+});
+
+// *** end of axios setup
+
+
+const loadAllProxyServers = async () => {
+    return {
+        'https': await loadProxyServersOfType('https'),
+        'socks5': await loadProxyServersOfType('socks5')
+    }
+}
+
+const loadProxyServersOfType = async (proxyType) => {
+    try {
+        const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
+        const serversList = response.body.split('\r\n');
+        return serversList.map(proxyServer => {
+            return {
+                address: proxyServer,
+                type: proxyType,
+                timeToFetch: undefined
+            }
+        });
+    }catch(e){
+        console.log('[ERROR](loadProxyServersOfType)', e);
+        return [];
+    }
+}
+
+const setTimeToFetch = async (proxyList) => {
+    const urlToFetch = process.env.URL_TO_FETCH.trim();
+    const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000;
+
+    return await Promise.map(proxyList, async ({ address, type }, i) => {
+        try{
+            if (address.length === 0) {
+                throw new Error('Malformed Proxy URL');
+            }
+            const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
+            let responsePromise;
+
+            if (type === 'https'){
+                // when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
+              responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout });
+            }else{
+              responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout });
+            }
+
+            // setting timeout in axios object is not enough for some requests
+            const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]);
+            return {
+                address,
+                type,
+                timeToFetch: responseObject.responseTime
+            }
+        }catch (e) {
+            // console.log('[ERROR](setTimeToFetch)', type, address);
+            return {
+                address,
+                type,
+                timeToFetch: undefined
+            }
+        }
+    }, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
+}
+
+exports.loadAllProxyServers = loadAllProxyServers;
+exports.setTimeToFetch = setTimeToFetch;