senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace hardcoded values with config in worker client

Browse Source
This commit is contained in:
Bilal
2020-05-08 07:51:02 +02:00
parent 87b18c7f7e
commit e279dda637
1 changed files with 8 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
workerclient/workerclient.go
View File

@@ -3,6 +3,7 @@ package workerclient
import ( import (
"bufio" "bufio"
b64 "encoding/base64" b64 "encoding/base64"
c "gitlab.com/saburly/kiviscraplib/config"
"gitlab.com/saburly/kiviscraplib/structures" "gitlab.com/saburly/kiviscraplib/structures"
"golang.org/x/net/proxy" "golang.org/x/net/proxy"
"io/ioutil" "io/ioutil"
@@ -17,8 +18,7 @@ import (
func StartClientConnections() { func StartClientConnections() {
rand.Seed(time.Now().Unix()) rand.Seed(time.Now().Unix())
// TODO: Take number of connections from ENV numberOfConnections := c.ClientConfig.ConnectionsCount
numberOfConnections := 1
for i := 0; i < numberOfConnections; i++ { for i := 0; i < numberOfConnections; i++ {
go startSingleConnection(i) go startSingleConnection(i)
@@ -32,12 +32,10 @@ func StartClientConnections() {
func startSingleConnection(connectionId int) { func startSingleConnection(connectionId int) {
log.Printf("(%d) Starting new client connection\n", connectionId) log.Printf("(%d) Starting new client connection\n", connectionId)
// TODO: Move initial connection timeout to the ENV connectionTimeout := c.ClientConfig.ConnectionTimeout
connectionTimeout := 2
for { for {
// TODO: Move server address to the ENV serverAddress := c.ClientConfig.WorkerServerAddress
const serverAddress = "127.0.0.1:1338"
conn, err := net.Dial("tcp", serverAddress) conn, err := net.Dial("tcp", serverAddress)
if err != nil { if err != nil {
log.Printf("(%d) Cannot connect to the load balancer server on %s : %s", connectionId, serverAddress, err) log.Printf("(%d) Cannot connect to the load balancer server on %s : %s", connectionId, serverAddress, err)
@@ -59,8 +57,7 @@ func startSingleConnection(connectionId int) {
log.Printf("(%d) Received new request message : %s", connectionId, requestMessage) log.Printf("(%d) Received new request message : %s", connectionId, requestMessage)
// TODO: Move prefix to the ENV requestMessagePrefix := c.ClientConfig.RequestMessagePrefix
const requestMessagePrefix = "URL "
if !strings.HasPrefix(requestMessage, requestMessagePrefix) { if !strings.HasPrefix(requestMessage, requestMessagePrefix) {
log.Printf("(%d) Request message has to start with %s", connectionId, requestMessagePrefix) log.Printf("(%d) Request message has to start with %s", connectionId, requestMessagePrefix)
_ = conn.Close() _ = conn.Close()
@@ -68,7 +65,7 @@ func startSingleConnection(connectionId int) {
break break
} }
urlToFetch := strings.TrimSuffix(strings.TrimPrefix(requestMessage, requestMessagePrefix), "\n") urlToFetch := strings.TrimSpace(strings.TrimPrefix(requestMessage, requestMessagePrefix))
pageBody, err := fetchPage(urlToFetch, connectionId) pageBody, err := fetchPage(urlToFetch, connectionId)
if err != nil { if err != nil {
@@ -181,7 +178,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
return []structures.ProxyServer{} return []structures.ProxyServer{}
} }
proxyListUrl := "https://www.proxy-list.download/api/v1/get?type=" + proxyType proxyListUrl := c.ClientConfig.ProxyListBaseURL + proxyType
resp, err := http.Get(proxyListUrl) resp, err := http.Get(proxyListUrl)
if err != nil { if err != nil {
@@ -197,7 +194,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
return []structures.ProxyServer{} return []structures.ProxyServer{}
} }
proxyAddresses := strings.Split(strings.TrimSuffix(string(proxyList), "\n"), "\n") proxyAddresses := strings.Split(strings.TrimSpace(string(proxyList)), "\n")
var result []structures.ProxyServer var result []structures.ProxyServer
for _, addr := range proxyAddresses { for _, addr := range proxyAddresses {