93 lines
3.0 KiB
JavaScript
93 lines
3.0 KiB
JavaScript
const Promise = require('bluebird');
|
|
const got = require('got');
|
|
const tunnel = require('tunnel');
|
|
const axios = require('axios').default;
|
|
const HttpsProxyAgent = require('https-proxy-agent');
|
|
const SocksProxyAgent = require('socks-proxy-agent');
|
|
|
|
// setup axios interceptors required to measure request times
|
|
|
|
axios.interceptors.request.use( x => {
|
|
// to avoid overwriting if another interceptor
|
|
// already defined the same object (meta)
|
|
x.meta = x.meta || {}
|
|
x.meta.requestStartedAt = new Date().getTime();
|
|
return x;
|
|
});
|
|
|
|
axios.interceptors.response.use(x => {
|
|
x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
|
|
return x;
|
|
});
|
|
|
|
// *** end of axios setup
|
|
|
|
|
|
const loadAllProxyServers = async () => {
|
|
return {
|
|
'https': await loadProxyServersOfType('https'),
|
|
'socks5': await loadProxyServersOfType('socks5')
|
|
}
|
|
}
|
|
|
|
const loadProxyServersOfType = async (proxyType) => {
|
|
try {
|
|
const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
|
|
const serversList = response.body.split('\r\n');
|
|
return serversList.map(proxyServer => {
|
|
return {
|
|
address: proxyServer,
|
|
type: proxyType,
|
|
timeToFetch: undefined
|
|
}
|
|
});
|
|
}catch(e){
|
|
console.log('[ERROR](loadProxyServersOfType)', e);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
const setTimeToFetch = async (proxyList) => {
|
|
const urlToFetch = process.env.URL_TO_FETCH.trim();
|
|
return await Promise.map(proxyList, async ({ address, type }, i) => {
|
|
console.log('Analyzing ', type, i, '/', proxyList.length);
|
|
try{
|
|
if (address.length === 0) {
|
|
throw new Error('Malformed Proxy URL');
|
|
}
|
|
const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
|
|
let responseObject = null;
|
|
if (type === 'https'){
|
|
// when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
|
|
responseObject = await axios({ httpAgent: proxyAgent, url: urlToFetch });
|
|
}else{
|
|
responseObject = await axios({ httpsAgent: proxyAgent, url: urlToFetch });
|
|
}
|
|
return {
|
|
address,
|
|
type,
|
|
timeToFetch: responseObject.responseTime
|
|
}
|
|
}catch (e) {
|
|
console.log('[ERROR](setTimeToFetch)', type, address, e);
|
|
return {
|
|
address,
|
|
type,
|
|
timeToFetch: undefined
|
|
}
|
|
}
|
|
}, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
|
|
}
|
|
|
|
const sortProxyServers = (proxyList) => {
|
|
return proxyList;
|
|
}
|
|
|
|
const selectBestProxies = (proxyList) => {
|
|
return proxyList;
|
|
}
|
|
|
|
exports.loadAllProxyServers = loadAllProxyServers;
|
|
exports.setTimeToFetch = setTimeToFetch;
|
|
exports.sortProxyServers = sortProxyServers;
|
|
exports.selectBestProxies = selectBestProxies; |