Accept URL as base64 string in scrape request

webserver/webserver.go

@@ -1,6 +1,7 @@
 package webserver
 
 import (
+	b64 "encoding/base64"
 	c "gitlab.com/saburly/kiviscraplib/config"
 	"gitlab.com/saburly/kiviscraplib/structures"
 	"gitlab.com/saburly/kiviscraplib/utils"
@@ -41,13 +42,16 @@ func httpHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if !utils.IsValidUrl(url[0]) {
+	decodedUrlBytes, _ := b64.StdEncoding.DecodeString(url[0])
+	decodedUrl := string(decodedUrlBytes)
+
+	if !utils.IsValidUrl(decodedUrl) {
 		respondWithError(w, 401, "url is malformed")
 		return
 	}
 
 	request := structures.Request{
-		Url:      url[0],
+		Url:      decodedUrl,
 		Response: make(chan structures.Response),
 	}