senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Include https proxies

Browse Source
This commit is contained in:
Bilal
2020-05-08 09:51:09 +02:00
parent e279dda637
commit 6d626ccecc
1 changed files with 34 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
workerclient/workerclient.go
View File

@@ -11,6 +11,7 @@ import (
"math/rand"
"net"
"net/http"
"net/url"
"strings"
"time"
)
@@ -91,31 +92,23 @@ func startSingleConnection(connectionId int) {
// fetchPage fetches page from desired url using random proxy
func fetchPage(url string, connectionId int) (string, error) {
dialer, proxyType, proxyAddr, err := getDialer()
// create request
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("(%d) [PROXY] Cannot create connection to the proxy %s %s : %s\n", connectionId, proxyType, proxyAddr, err)
log.Printf("(%d) Cannot create GET request to the %s : %s\n", connectionId, url, err)
return "", err
}
log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, proxyType, proxyAddr)
// setup a http client
httpTransport := &http.Transport{}
httpClient := &http.Client{Transport: httpTransport}
httpTransport.Dial = dialer.Dial
// create
req, err := http.NewRequest("GET", url, nil)
// create httpClient to execute request
httpClient, err := getHttpClient(connectionId)
if err != nil {
log.Printf("(%d) [PROXY] Cannot create GET request to the %s : %s", connectionId, url, err)
return "", err
}
// use the http client to fetch the page
resp, err2 := httpClient.Do(req)
if err2 != nil {
log.Printf("(%d) [PROXY] Cannot GET page %s : %s", connectionId, url, err2)
log.Printf("(%d) [PROXY] Cannot GET page %s : %s\n", connectionId, url, err2)
return "", err2
}
@@ -124,47 +117,58 @@ func fetchPage(url string, connectionId int) (string, error) {
pageBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("(%d) [PROXY] Cannot read body of %s : %s", connectionId, url, err)
log.Printf("(%d) [PROXY] Cannot read body of %s : %s\n", connectionId, url, err)
return "", err
}
return string(pageBody), nil
}
// getDialer selects one random proxy from the slice of all proxies
func getDialer() (proxy.Dialer, string, string, error) {
// getHttpClient creates client with proxy connection; Proxy is selected randomly from list of proxies
func getHttpClient(connectionId int) (*http.Client, error) {
proxyList := getAllProxies()
// setup a http client
httpTransport := &http.Transport{}
httpClient := &http.Client{Transport: httpTransport}
if len(proxyList) == 0 {
return proxy.Direct, "", "", nil // No proxy
log.Printf("(%d) [PROXY] No proxy found, will continue without proxy!\n", connectionId)
return httpClient, nil
}
// get random proxy from the list
selectedProxy := proxyList[rand.Intn(len(proxyList))]
if selectedProxy.Type == "https" {
// TODO: Implement https proxy
//parsedProxyURL, err := url.Parse("https://" + selectedProxy.Address)
//if err != nil {
// return nil, "", "", err
//}
//
//dialer, err := proxy.FromURL(parsedProxyURL, proxy.Direct)
//return dialer, selectedProxy.Type, selectedProxy.Address, err
proxyUrl, err := url.Parse("http://" + selectedProxy.Address)
if err != nil {
log.Printf("(%d) [PROXY] Cannot parse proxy address (%s) %s : %s\n",
connectionId, selectedProxy.Type, selectedProxy.Address, err)
return nil, err
}
httpTransport.Proxy = http.ProxyURL(proxyUrl)
//myClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
}
if selectedProxy.Type == "socks5" {
dialer, err := proxy.SOCKS5("tcp", selectedProxy.Address, nil, proxy.Direct)
return dialer, selectedProxy.Type, selectedProxy.Address, err
if err != nil {
log.Printf("(%d) [PROXY] Cannot create connection to the proxy (%s) %s : %s\n",
connectionId, selectedProxy.Type, selectedProxy.Address, err)
return nil, err
}
httpTransport.Dial = dialer.Dial
}
return proxy.Direct, "", "", nil
log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, selectedProxy.Type, selectedProxy.Address)
return httpClient, nil
}
// getAllProxies combines all proxy types in one slice
func getAllProxies() []structures.ProxyServer {
//return append(getProxiesList("socks5"), getProxiesList("https")...)
return getProxiesList("socks5")
return append(getProxiesList("socks5"), getProxiesList("https")...)
}
// getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https")