Merge branch 'skip-proxy-on-failure' into 'master'

Add custom proxies from ENV

See merge request saburly/kiviscraplib!4

config/config.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"strconv"
+	"strings"
 )
 
 var WebServerConfig structures.WebServerConfig
@@ -43,6 +44,16 @@ func generateClientConfigObject() {
 	ClientConfig.WorkerServerAddress = getString("WORKER_SERVER_ADDRESS")
 	ClientConfig.RequestMessagePrefix = getString("REQUEST_MESSAGE_PREFIX")
 	ClientConfig.ProxyListBaseURL = getString("PROXY_LIST_BASE_URL")
+
+	customSOCKS5ProxyListString := getString("CUSTOM_SOCKS5_PROXY_LIST")
+	customSOCKS5ProxyList := strings.Split(customSOCKS5ProxyListString, ",")
+
+	for i := range customSOCKS5ProxyList {
+		proxy := structures.ProxyServer{Type: "socks5", Address: strings.TrimSpace(customSOCKS5ProxyList[i])}
+		if len(proxy.Address) > 0 {
+			ClientConfig.CustomSOCKS5ProxyList = append(ClientConfig.CustomSOCKS5ProxyList, proxy)
+		}
+	}
 }
 
 func generateServerConfigObject() {
@@ -61,7 +72,8 @@ func initClientConfigDefaultValues() {
 	defaultClientConfigValues["CLIENT_WAITING_TIMEOUT"] = "60"
 	defaultClientConfigValues["WORKER_SERVER_ADDRESS"] = "127.0.0.1:1338"
 	defaultClientConfigValues["REQUEST_MESSAGE_PREFIX"] = "URL "
-	defaultClientConfigValues["PROXY_LIST_BASE_URL"] = "https://www.proxy-list.download/api/v1/get?type="
+	defaultClientConfigValues["PROXY_LIST_BASE_URL"] = ""
+	defaultClientConfigValues["CUSTOM_SOCKS5_PROXY_LIST"] = ""
 }
 
 func initServerConfigDefaultValues() {

structures/structures.go

@@ -34,10 +34,11 @@ type WorkerServerConfig struct {
 }
 
 type ClientConfig struct {
-	ConnectionsCount     int
+	ConnectionsCount      int
-	ConnectionTimeout    int // In seconds
+	ConnectionTimeout     int // In seconds
-	WaitingTimeout       int // In seconds
+	WaitingTimeout        int // In seconds
-	WorkerServerAddress  string
+	WorkerServerAddress   string
-	RequestMessagePrefix string
+	RequestMessagePrefix  string
-	ProxyListBaseURL     string
+	ProxyListBaseURL      string
+	CustomSOCKS5ProxyList []ProxyServer
 }

workerclient/workerclient.go

@@ -196,11 +196,22 @@ func getHttpClient(connectionId int) (*http.Client, error) {
 
 // getAllProxies combines all proxy types in one slice
 func getAllProxies() []structures.ProxyServer {
-	return append(getProxiesList("socks5"), getProxiesList("https")...)
+	var result []structures.ProxyServer
+
+	result = append(result, getProxiesList("socks5")...)
+	result = append(result, getProxiesList("https")...)
+
+	result = append(result, c.ClientConfig.CustomSOCKS5ProxyList...)
+
+	return result
 }
 
 // getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https")
 func getProxiesList(proxyType string) []structures.ProxyServer {
+	if len(c.ClientConfig.ProxyListBaseURL) == 0 {
+		return []structures.ProxyServer{}
+	}
+
 	switch proxyType {
 	case "https":
 	case "socks5":
@@ -226,6 +237,12 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
 		return []structures.ProxyServer{}
 	}
 
+	if strings.Contains(string(proxyList), "html") {
+		// Something is wrong, expected to receive proxy list but something else is returned
+		log.Println("Proxy list malformed")
+		return []structures.ProxyServer{}
+	}
+
 	proxyAddresses := strings.Split(strings.TrimSpace(string(proxyList)), "\n")
 
 	var result []structures.ProxyServer