Adapt worker server for base64 client response

workerserver/workerserver.go

@@ -2,10 +2,12 @@ package workerserver
 
 import (
 	"bufio"
+	b64 "encoding/base64"
 	"gitlab.com/saburly/kiviscraplib/structures"
 	"log"
 	"math/rand"
 	"net"
+	"strings"
 	"time"
 )
 
@@ -53,6 +55,8 @@ func handleConnection(conn net.Conn) {
 		Req: make(chan structures.Request),
 	}
 
+	log.Printf("New worker client connected from %s\n", clientAddr)
+
 	workers <- workerDescription // add new worker to the end of the queue
 	for {
 		request := <-workerDescription.Req
@@ -65,7 +69,7 @@ func handleConnection(conn net.Conn) {
 			return              // don't return worker to the queue - not healthy
 		}
 
-		bufferBytes, err := bufio.NewReader(conn).ReadBytes('\n')
+		bufferString, err := bufio.NewReader(conn).ReadString('\n')
 		if err != nil {
 			log.Println("client left..")
 			conn.Close()
@@ -73,10 +77,12 @@ func handleConnection(conn net.Conn) {
 			return              // don't return worker to the queue - not healthy
 		}
 
+		decodedPageBody, _ := b64.StdEncoding.DecodeString(strings.TrimSpace(bufferString))
+
 		log.Printf("Recieving response from %s\n", clientAddr)
 		response := structures.Response{
 			Url:     request.Url,
-			Content: bufferBytes,
+			Content: decodedPageBody,
 			Err:     nil,
 		}