senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

handle invalid response from proxy-list; add custom proxies from ENV

Browse Source
This commit is contained in:
Bilal
2020-05-19 19:14:21 +02:00
parent eb6bcb981d
commit 4779f0a91e
3 changed files with 38 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
workerclient/workerclient.go
View File

@@ -196,11 +196,22 @@ func getHttpClient(connectionId int) (*http.Client, error) {
// getAllProxies combines all proxy types in one slice
func getAllProxies() []structures.ProxyServer {
return append(getProxiesList("socks5"), getProxiesList("https")...)
var result []structures.ProxyServer
result = append(result, getProxiesList("socks5")...)
result = append(result, getProxiesList("https")...)
result = append(result, c.ClientConfig.CustomSOCKS5ProxyList...)
return result
}
// getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https")
func getProxiesList(proxyType string) []structures.ProxyServer {
if len(c.ClientConfig.ProxyListBaseURL) == 0 {
return []structures.ProxyServer{}
}
switch proxyType {
case "https":
case "socks5":
@@ -226,6 +237,12 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
return []structures.ProxyServer{}
}
if strings.Contains(string(proxyList), "html") {
// Something is wrong, expected to receive proxy list but something else is returned
log.Println("Proxy list malformed")
return []structures.ProxyServer{}
}
proxyAddresses := strings.Split(strings.TrimSpace(string(proxyList)), "\n")
var result []structures.ProxyServer