senaduka/old-kiviscraplib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace hardcoded values with config in worker client

Browse Source
This commit is contained in:
Bilal
2020-05-08 07:51:02 +02:00
parent 87b18c7f7e
commit e279dda637
1 changed files with 8 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
workerclient/workerclient.go
View File

@@ -3,6 +3,7 @@ package workerclient
import (
"bufio"
b64 "encoding/base64"
c "gitlab.com/saburly/kiviscraplib/config"
"gitlab.com/saburly/kiviscraplib/structures"
"golang.org/x/net/proxy"
"io/ioutil"
@@ -17,8 +18,7 @@ import (
func StartClientConnections() {
rand.Seed(time.Now().Unix())
// TODO: Take number of connections from ENV
numberOfConnections := 1
numberOfConnections := c.ClientConfig.ConnectionsCount
for i := 0; i < numberOfConnections; i++ {
go startSingleConnection(i)
@@ -32,12 +32,10 @@ func StartClientConnections() {
func startSingleConnection(connectionId int) {
log.Printf("(%d) Starting new client connection\n", connectionId)
// TODO: Move initial connection timeout to the ENV
connectionTimeout := 2
connectionTimeout := c.ClientConfig.ConnectionTimeout
for {
// TODO: Move server address to the ENV
const serverAddress = "127.0.0.1:1338"
serverAddress := c.ClientConfig.WorkerServerAddress
conn, err := net.Dial("tcp", serverAddress)
if err != nil {
log.Printf("(%d) Cannot connect to the load balancer server on %s : %s", connectionId, serverAddress, err)
@@ -59,8 +57,7 @@ func startSingleConnection(connectionId int) {
log.Printf("(%d) Received new request message : %s", connectionId, requestMessage)
// TODO: Move prefix to the ENV
const requestMessagePrefix = "URL "
requestMessagePrefix := c.ClientConfig.RequestMessagePrefix
if !strings.HasPrefix(requestMessage, requestMessagePrefix) {
log.Printf("(%d) Request message has to start with %s", connectionId, requestMessagePrefix)
_ = conn.Close()
@@ -68,7 +65,7 @@ func startSingleConnection(connectionId int) {
break
}
urlToFetch := strings.TrimSuffix(strings.TrimPrefix(requestMessage, requestMessagePrefix), "\n")
urlToFetch := strings.TrimSpace(strings.TrimPrefix(requestMessage, requestMessagePrefix))
pageBody, err := fetchPage(urlToFetch, connectionId)
if err != nil {
@@ -181,7 +178,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
return []structures.ProxyServer{}
}
proxyListUrl := "https://www.proxy-list.download/api/v1/get?type=" + proxyType
proxyListUrl := c.ClientConfig.ProxyListBaseURL + proxyType
resp, err := http.Get(proxyListUrl)
if err != nil {
@@ -197,7 +194,7 @@ func getProxiesList(proxyType string) []structures.ProxyServer {
return []structures.ProxyServer{}
}
proxyAddresses := strings.Split(strings.TrimSuffix(string(proxyList), "\n"), "\n")
proxyAddresses := strings.Split(strings.TrimSpace(string(proxyList)), "\n")
var result []structures.ProxyServer
for _, addr := range proxyAddresses {