senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Include https proxies

Browse Source
This commit is contained in:
Bilal
2020-05-08 09:51:09 +02:00
parent e279dda637
commit 6d626ccecc
1 changed files with 34 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
workerclient/workerclient.go
View File

@@ -11,6 +11,7 @@ import (
"math/rand" "math/rand"
"net" "net"
"net/http" "net/http"
"net/url"
"strings" "strings"
"time" "time"
) )
@@ -91,31 +92,23 @@ func startSingleConnection(connectionId int) {
// fetchPage fetches page from desired url using random proxy // fetchPage fetches page from desired url using random proxy
func fetchPage(url string, connectionId int) (string, error) { func fetchPage(url string, connectionId int) (string, error) {
dialer, proxyType, proxyAddr, err := getDialer() // create request
req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
log.Printf("(%d) [PROXY] Cannot create connection to the proxy %s %s : %s\n", connectionId, proxyType, proxyAddr, err) log.Printf("(%d) Cannot create GET request to the %s : %s\n", connectionId, url, err)
return "", err return "", err
} }
log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, proxyType, proxyAddr) // create httpClient to execute request
httpClient, err := getHttpClient(connectionId)
// setup a http client
httpTransport := &http.Transport{}
httpClient := &http.Client{Transport: httpTransport}
httpTransport.Dial = dialer.Dial
// create
req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
log.Printf("(%d) [PROXY] Cannot create GET request to the %s : %s", connectionId, url, err)
return "", err return "", err
} }
// use the http client to fetch the page // use the http client to fetch the page
resp, err2 := httpClient.Do(req) resp, err2 := httpClient.Do(req)
if err2 != nil { if err2 != nil {
log.Printf("(%d) [PROXY] Cannot GET page %s : %s", connectionId, url, err2) log.Printf("(%d) [PROXY] Cannot GET page %s : %s\n", connectionId, url, err2)
return "", err2 return "", err2
} }
@@ -124,47 +117,58 @@ func fetchPage(url string, connectionId int) (string, error) {
pageBody, err := ioutil.ReadAll(resp.Body) pageBody, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Printf("(%d) [PROXY] Cannot read body of %s : %s", connectionId, url, err) log.Printf("(%d) [PROXY] Cannot read body of %s : %s\n", connectionId, url, err)
return "", err return "", err
} }
return string(pageBody), nil return string(pageBody), nil
} }
// getDialer selects one random proxy from the slice of all proxies // getHttpClient creates client with proxy connection; Proxy is selected randomly from list of proxies
func getDialer() (proxy.Dialer, string, string, error) { func getHttpClient(connectionId int) (*http.Client, error) {
proxyList := getAllProxies() proxyList := getAllProxies()
// setup a http client
httpTransport := &http.Transport{}
httpClient := &http.Client{Transport: httpTransport}
if len(proxyList) == 0 { if len(proxyList) == 0 {
return proxy.Direct, "", "", nil // No proxy log.Printf("(%d) [PROXY] No proxy found, will continue without proxy!\n", connectionId)
return httpClient, nil
} }
// get random proxy from the list // get random proxy from the list
selectedProxy := proxyList[rand.Intn(len(proxyList))] selectedProxy := proxyList[rand.Intn(len(proxyList))]
if selectedProxy.Type == "https" { if selectedProxy.Type == "https" {
// TODO: Implement https proxy proxyUrl, err := url.Parse("http://" + selectedProxy.Address)
//parsedProxyURL, err := url.Parse("https://" + selectedProxy.Address) if err != nil {
//if err != nil { log.Printf("(%d) [PROXY] Cannot parse proxy address (%s) %s : %s\n",
// return nil, "", "", err connectionId, selectedProxy.Type, selectedProxy.Address, err)
//} return nil, err
// }
//dialer, err := proxy.FromURL(parsedProxyURL, proxy.Direct) httpTransport.Proxy = http.ProxyURL(proxyUrl)
//return dialer, selectedProxy.Type, selectedProxy.Address, err //myClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
} }
if selectedProxy.Type == "socks5" { if selectedProxy.Type == "socks5" {
dialer, err := proxy.SOCKS5("tcp", selectedProxy.Address, nil, proxy.Direct) dialer, err := proxy.SOCKS5("tcp", selectedProxy.Address, nil, proxy.Direct)
return dialer, selectedProxy.Type, selectedProxy.Address, err if err != nil {
log.Printf("(%d) [PROXY] Cannot create connection to the proxy (%s) %s : %s\n",
connectionId, selectedProxy.Type, selectedProxy.Address, err)
return nil, err
}
httpTransport.Dial = dialer.Dial
} }
return proxy.Direct, "", "", nil log.Printf("(%d) [PROXY] Selected proxy (%s) %s\n", connectionId, selectedProxy.Type, selectedProxy.Address)
return httpClient, nil
} }
// getAllProxies combines all proxy types in one slice // getAllProxies combines all proxy types in one slice
func getAllProxies() []structures.ProxyServer { func getAllProxies() []structures.ProxyServer {
//return append(getProxiesList("socks5"), getProxiesList("https")...) return append(getProxiesList("socks5"), getProxiesList("https")...)
return getProxiesList("socks5")
} }
// getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https") // getProxiesList fetches list of proxies of specific type (valid types are : "socks5", "https")