diff --git a/structures/structures.go b/structures/structures.go new file mode 100644 index 0000000..894b35a --- /dev/null +++ b/structures/structures.go @@ -0,0 +1,12 @@ +package structures + +type Response struct { + Url string + Content []byte + Err error +} + +type Request struct { + Url string + Response chan Response +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..74a44aa --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,20 @@ +package utils + +import ( + "net/url" +) + +// IsValidUrl tests a string to determine if it is a well-structured url or not. +func IsValidUrl(toTest string) bool { + _, err := url.ParseRequestURI(toTest) + if err != nil { + return false + } + + u, err := url.Parse(toTest) + if err != nil || u.Scheme == "" || u.Host == "" { + return false + } + + return true +} diff --git a/webserver/webserver.go b/webserver/webserver.go new file mode 100644 index 0000000..f201a59 --- /dev/null +++ b/webserver/webserver.go @@ -0,0 +1,100 @@ +package webserver + +import ( + "gitlab.com/saburly/kiviscraplib/structures" + "gitlab.com/saburly/kiviscraplib/utils" + "log" + "net/http" + "time" +) + +var requests chan<- structures.Request + +func ServeHTTP(queue chan<- structures.Request, end chan<- string) { + requests = queue + http.HandleFunc("/", httpHandler) + err := http.ListenAndServe("127.0.0.1:1337", nil) + if err != nil { + end <- "http server" + log.Fatal(err) + } +} + +func httpHandler(w http.ResponseWriter, r *http.Request) { + + query := r.URL.Query() + api_key, present := query["api_key"] + if !present || len(api_key) == 0 { + respondWithError(w, 400, "api_key is missing") + return + } + + // todo: do more for the authentication / authorization + if api_key[0] != "b8be5a3b639d465039e8fbe7582270a7" { + respondWithError(w, 401, "api_key is wrong") + return + } + + url, present := query["url"] + if !present || len(url) == 0 { + respondWithError(w, 400, "url is missing") + return + } + + if !utils.IsValidUrl(url[0]) { + respondWithError(w, 401, "url is malformed") + return + } + + request := structures.Request{ + Url: url[0], + Response: make(chan structures.Response), + } + + requests <- request + + log.Printf("request from %s: %s %q", r.RemoteAddr, r.Method, r.URL) + log.Printf("waiting for response") + + // todo: put timeout in ENV variable + timeout := make(chan bool, 1) + go func() { + time.Sleep(10 * time.Second) + timeout <- true + }() + + timeoutResponse := false + response := structures.Response{} + + select { + case response = <-request.Response: + timeoutResponse = false + case <-timeout: + timeoutResponse = true + } + + if timeoutResponse { + log.Printf("response timed out") + respondWithError(w, 500, "fetching timeout") + return + } + + if response.Err != nil { + log.Printf("error with responding") + respondWithError(w, 500, "error fetching") + return + } + + log.Printf("response successful!") + respondWithContent(w, response.Content) +} + +func respondWithError(w http.ResponseWriter, status int, message string) { + w.WriteHeader(status) + w.Write([]byte(message)) +} + +func respondWithContent(w http.ResponseWriter, content []byte) { + w.WriteHeader(200) + w.Write(content) +}