diff --git a/workerclient/workerclient.go b/workerclient/workerclient.go index ac7682d..720e1ee 100644 --- a/workerclient/workerclient.go +++ b/workerclient/workerclient.go @@ -3,6 +3,7 @@ package workerclient import ( "bufio" b64 "encoding/base64" + c "gitlab.com/saburly/kiviscraplib/config" "gitlab.com/saburly/kiviscraplib/structures" "golang.org/x/net/proxy" "io/ioutil" @@ -17,8 +18,7 @@ import ( func StartClientConnections() { rand.Seed(time.Now().Unix()) - // TODO: Take number of connections from ENV - numberOfConnections := 1 + numberOfConnections := c.ClientConfig.ConnectionsCount for i := 0; i < numberOfConnections; i++ { go startSingleConnection(i) @@ -32,12 +32,10 @@ func StartClientConnections() { func startSingleConnection(connectionId int) { log.Printf("(%d) Starting new client connection\n", connectionId) - // TODO: Move initial connection timeout to the ENV - connectionTimeout := 2 + connectionTimeout := c.ClientConfig.ConnectionTimeout for { - // TODO: Move server address to the ENV - const serverAddress = "127.0.0.1:1338" + serverAddress := c.ClientConfig.WorkerServerAddress conn, err := net.Dial("tcp", serverAddress) if err != nil { log.Printf("(%d) Cannot connect to the load balancer server on %s : %s", connectionId, serverAddress, err) @@ -59,8 +57,7 @@ func startSingleConnection(connectionId int) { log.Printf("(%d) Received new request message : %s", connectionId, requestMessage) - // TODO: Move prefix to the ENV - const requestMessagePrefix = "URL " + requestMessagePrefix := c.ClientConfig.RequestMessagePrefix if !strings.HasPrefix(requestMessage, requestMessagePrefix) { log.Printf("(%d) Request message has to start with %s", connectionId, requestMessagePrefix) _ = conn.Close() @@ -68,7 +65,7 @@ func startSingleConnection(connectionId int) { break } - urlToFetch := strings.TrimSuffix(strings.TrimPrefix(requestMessage, requestMessagePrefix), "\n") + urlToFetch := strings.TrimSpace(strings.TrimPrefix(requestMessage, requestMessagePrefix)) pageBody, err := fetchPage(urlToFetch, connectionId) if err != nil { @@ -181,7 +178,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer { return []structures.ProxyServer{} } - proxyListUrl := "https://www.proxy-list.download/api/v1/get?type=" + proxyType + proxyListUrl := c.ClientConfig.ProxyListBaseURL + proxyType resp, err := http.Get(proxyListUrl) if err != nil { @@ -197,7 +194,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer { return []structures.ProxyServer{} } - proxyAddresses := strings.Split(strings.TrimSuffix(string(proxyList), "\n"), "\n") + proxyAddresses := strings.Split(strings.TrimSpace(string(proxyList)), "\n") var result []structures.ProxyServer for _, addr := range proxyAddresses {