senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add timeout on proxy connections

Browse Source
This commit is contained in:
Bilal
2020-05-22 16:08:59 +02:00
parent 5fb5d22ff2
commit 0772c2ba1b
3 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.go
View File

@@ -46,6 +46,7 @@ func generateClientConfigObject() {
ClientConfig.ProxyListBaseURL = getString("PROXY_LIST_BASE_URL") ClientConfig.ProxyListBaseURL = getString("PROXY_LIST_BASE_URL")
ClientConfig.ProxyListReloadInterval = getInt("PROXY_LIST_RELOAD_INTERVAL") ClientConfig.ProxyListReloadInterval = getInt("PROXY_LIST_RELOAD_INTERVAL")
ClientConfig.ProxyListTimeout = getInt("PROXY_LIST_TIMEOUT") ClientConfig.ProxyListTimeout = getInt("PROXY_LIST_TIMEOUT")
ClientConfig.FetchTimeout = getInt("FETCH_TIMEOUT")
customSOCKS5ProxyListString := getString("CUSTOM_SOCKS5_PROXY_LIST") customSOCKS5ProxyListString := getString("CUSTOM_SOCKS5_PROXY_LIST")
customSOCKS5ProxyList := strings.Split(customSOCKS5ProxyListString, ",") customSOCKS5ProxyList := strings.Split(customSOCKS5ProxyListString, ",")
@@ -78,6 +79,7 @@ func initClientConfigDefaultValues() {
defaultClientConfigValues["CUSTOM_SOCKS5_PROXY_LIST"] = "" defaultClientConfigValues["CUSTOM_SOCKS5_PROXY_LIST"] = ""
defaultClientConfigValues["PROXY_LIST_RELOAD_INTERVAL"] = "30" defaultClientConfigValues["PROXY_LIST_RELOAD_INTERVAL"] = "30"
defaultClientConfigValues["PROXY_LIST_TIMEOUT"] = "10" defaultClientConfigValues["PROXY_LIST_TIMEOUT"] = "10"
defaultClientConfigValues["FETCH_TIMEOUT"] = "60"
} }
func initServerConfigDefaultValues() { func initServerConfigDefaultValues() {

1
structures/structures.go
View File

@@ -43,4 +43,5 @@ type ClientConfig struct {
ProxyListTimeout int // In seconds ProxyListTimeout int // In seconds
ProxyListReloadInterval int // In minutes ProxyListReloadInterval int // In minutes
CustomSOCKS5ProxyList []ProxyServer CustomSOCKS5ProxyList []ProxyServer
FetchTimeout int // In seconds
} }

5
workerclient/workerclient.go
View File

@@ -170,7 +170,10 @@ func fetchPage(url string, connectionId int) (string, error) {
func getHttpClient(connectionId int) (*http.Client, error) { func getHttpClient(connectionId int) (*http.Client, error) {
// setup a http client // setup a http client
httpTransport := &http.Transport{} httpTransport := &http.Transport{}
httpClient := &http.Client{Transport: httpTransport} httpClient := &http.Client{
Transport: httpTransport,
Timeout: time.Duration(c.ClientConfig.FetchTimeout) * time.Second,
}
if len(proxyList) == 0 { if len(proxyList) == 0 {
log.Printf("(%d) [PROXY] No proxy found, will continue without proxy!\n", connectionId) log.Printf("(%d) [PROXY] No proxy found, will continue without proxy!\n", connectionId)