90 lines
3.0 KiB
JavaScript
90 lines
3.0 KiB
JavaScript
const Promise = require('bluebird');
|
|
const got = require('got');
|
|
const tunnel = require('tunnel');
|
|
const axios = require('axios').default;
|
|
const HttpsProxyAgent = require('https-proxy-agent');
|
|
const SocksProxyAgent = require('socks-proxy-agent');
|
|
|
|
const { timeoutPromise } = require('./helpers');
|
|
|
|
// setup axios interceptors required to measure request times and timeout
|
|
|
|
axios.interceptors.request.use( x => {
|
|
// to avoid overwriting if another interceptor
|
|
// already defined the same object (meta)
|
|
x.meta = x.meta || {}
|
|
x.meta.requestStartedAt = new Date().getTime();
|
|
return x;
|
|
});
|
|
|
|
axios.interceptors.response.use(x => {
|
|
x.responseTime = new Date().getTime() - x.config.meta.requestStartedAt;
|
|
return x;
|
|
});
|
|
|
|
// *** end of axios setup
|
|
|
|
|
|
const loadAllProxyServers = async () => {
|
|
return {
|
|
'https': await loadProxyServersOfType('https'),
|
|
'socks5': await loadProxyServersOfType('socks5')
|
|
}
|
|
}
|
|
|
|
const loadProxyServersOfType = async (proxyType) => {
|
|
try {
|
|
const response = await got(`${process.env.PROXY_LIST_BASE_URL}${proxyType}`, { timeout: process.env.PROXY_LIST_TIMEOUT*1000 });
|
|
const serversList = response.body.split('\r\n');
|
|
return serversList.map(proxyServer => {
|
|
return {
|
|
address: proxyServer,
|
|
type: proxyType,
|
|
timeToFetch: undefined
|
|
}
|
|
});
|
|
}catch(e){
|
|
console.log('[ERROR](loadProxyServersOfType)', e);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
const setTimeToFetch = async (proxyList) => {
|
|
const urlToFetch = process.env.URL_TO_FETCH.trim();
|
|
const timeout = parseInt(process.env.SINGLE_PROXY_TIMEOUT)*1000;
|
|
|
|
return await Promise.map(proxyList, async ({ address, type }, i) => {
|
|
try{
|
|
if (address.length === 0) {
|
|
throw new Error('Malformed Proxy URL');
|
|
}
|
|
const proxyAgent = (type === 'https') ? new HttpsProxyAgent(`https://${address}`) : new SocksProxyAgent(`socks5://${address}`);
|
|
let responsePromise;
|
|
|
|
if (type === 'https'){
|
|
// when using HTTPS, set 'httpAgent' instead of 'httpsAgent'
|
|
responsePromise = axios({ httpAgent: proxyAgent, url: urlToFetch, timeout });
|
|
}else{
|
|
responsePromise = axios({ httpsAgent: proxyAgent, url: urlToFetch, timeout });
|
|
}
|
|
|
|
// setting timeout in axios object is not enough for some requests
|
|
const responseObject = await Promise.race([responsePromise, timeoutPromise(timeout)]);
|
|
return {
|
|
address,
|
|
type,
|
|
timeToFetch: responseObject.responseTime
|
|
}
|
|
}catch (e) {
|
|
console.log('[ERROR](setTimeToFetch)', type, address);
|
|
return {
|
|
address,
|
|
type,
|
|
timeToFetch: undefined
|
|
}
|
|
}
|
|
}, { concurrency: parseInt(process.env.MAX_PROXY_CONNECTIONS) || 5});
|
|
}
|
|
|
|
exports.loadAllProxyServers = loadAllProxyServers;
|
|
exports.setTimeToFetch = setTimeToFetch; |