diff --git a/workerclient/workerclient.go b/workerclient/workerclient.go index 720e1ee..b82f4f7 100644 --- a/workerclient/workerclient.go +++ b/workerclient/workerclient.go @@ -11,6 +11,7 @@ import ( "math/rand" "net" "net/http" + "net/url" "strings" "time" ) @@ -91,31 +92,23 @@ func startSingleConnection(connectionId int) { // fetchPage fetches page from desired url using random proxy func fetchPage(url string, connectionId int) (string, error) { - dialer, proxyType, proxyAddr, err := getDialer() + // create request + req, err := http.NewRequest("GET", url, nil) if err != nil { - log.Printf("(%d) [PROXY] Cannot create connection to the proxy %s %s : %s\n", connectionId, proxyType, proxyAddr, err) + log.Printf("(%d) Cannot create GET request to the %s : %s\n", connectionId, url, err) return "", err } - log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, proxyType, proxyAddr) - - // setup a http client - httpTransport := &http.Transport{} - httpClient := &http.Client{Transport: httpTransport} - - httpTransport.Dial = dialer.Dial - - // create - req, err := http.NewRequest("GET", url, nil) + // create httpClient to execute request + httpClient, err := getHttpClient(connectionId) if err != nil { - log.Printf("(%d) [PROXY] Cannot create GET request to the %s : %s", connectionId, url, err) return "", err } // use the http client to fetch the page resp, err2 := httpClient.Do(req) if err2 != nil { - log.Printf("(%d) [PROXY] Cannot GET page %s : %s", connectionId, url, err2) + log.Printf("(%d) [PROXY] Cannot GET page %s : %s\n", connectionId, url, err2) return "", err2 } @@ -124,47 +117,58 @@ func fetchPage(url string, connectionId int) (string, error) { pageBody, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Printf("(%d) [PROXY] Cannot read body of %s : %s", connectionId, url, err) + log.Printf("(%d) [PROXY] Cannot read body of %s : %s\n", connectionId, url, err) return "", err } return string(pageBody), nil } -// getDialer selects one random proxy from the slice of all proxies -func getDialer() (proxy.Dialer, string, string, error) { +// getHttpClient creates client with proxy connection; Proxy is selected randomly from list of proxies +func getHttpClient(connectionId int) (*http.Client, error) { proxyList := getAllProxies() + // setup a http client + httpTransport := &http.Transport{} + httpClient := &http.Client{Transport: httpTransport} + if len(proxyList) == 0 { - return proxy.Direct, "", "", nil // No proxy + log.Printf("(%d) [PROXY] No proxy found, will continue without proxy!\n", connectionId) + return httpClient, nil } // get random proxy from the list selectedProxy := proxyList[rand.Intn(len(proxyList))] if selectedProxy.Type == "https" { - // TODO: Implement https proxy - //parsedProxyURL, err := url.Parse("https://" + selectedProxy.Address) - //if err != nil { - // return nil, "", "", err - //} - // - //dialer, err := proxy.FromURL(parsedProxyURL, proxy.Direct) - //return dialer, selectedProxy.Type, selectedProxy.Address, err + proxyUrl, err := url.Parse("http://" + selectedProxy.Address) + if err != nil { + log.Printf("(%d) [PROXY] Cannot parse proxy address (%s) %s : %s\n", + connectionId, selectedProxy.Type, selectedProxy.Address, err) + return nil, err + } + httpTransport.Proxy = http.ProxyURL(proxyUrl) + //myClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} } if selectedProxy.Type == "socks5" { dialer, err := proxy.SOCKS5("tcp", selectedProxy.Address, nil, proxy.Direct) - return dialer, selectedProxy.Type, selectedProxy.Address, err + if err != nil { + log.Printf("(%d) [PROXY] Cannot create connection to the proxy (%s) %s : %s\n", + connectionId, selectedProxy.Type, selectedProxy.Address, err) + return nil, err + } + httpTransport.Dial = dialer.Dial } - return proxy.Direct, "", "", nil + log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, selectedProxy.Type, selectedProxy.Address) + + return httpClient, nil } // getAllProxies combines all proxy types in one slice func getAllProxies() []structures.ProxyServer { - //return append(getProxiesList("socks5"), getProxiesList("https")...) - return getProxiesList("socks5") + return append(getProxiesList("socks5"), getProxiesList("https")...) } // getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https")