From 9a8f89d16b2ee204894d15034552a1d9402bf3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Senart?= Date: Fri, 4 Oct 2013 21:46:16 +0200 Subject: [PATCH] Extend Metrics with max and 95th, 99th percentiles Implements a new format for the text reporter with more information than before. Also Rename Result.Timing to Result.Latency and related names. Adapt README.md examples to account new text reporter format and library changes. --- README.md | 30 ++++++++++----------- lib/attack.go | 2 +- lib/metrics.go | 63 ++++++++++++++++++++++++++++++--------------- lib/metrics_test.go | 20 +++++++------- lib/reporters.go | 24 ++++++++--------- lib/results.go | 2 +- 6 files changed, 81 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index b2bb0cf..a93c002 100644 --- a/README.md +++ b/README.md @@ -117,16 +117,21 @@ Specifies the kind of report to be generated. It defaults to text. ##### text ``` -Time(avg) Requests Success Bytes(rx/tx) -152.341ms 200 17.00% 251.00/0.00 - -Count: 49 30 39 48 34 -Status: 500 404 409 503 200 - +Requests [total] 1200 +Latencies [mean, max, 95, 99] 223.340085ms, 7.788103259s, 326.913687ms, 416.537743ms +Bytes In [total, mean] 3714690, 3095.57 +Bytes Out [total, mean] 0, 0.00 +Success [ratio] 55.42% +Status Codes [code:count] 0:535 200:665 Error Set: -Server Timeout -Page Not Found +Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection refused +Get http://localhost:6060: read tcp 127.0.0.1:6060: connection reset by peer +Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection reset by peer +Get http://localhost:6060: write tcp 127.0.0.1:6060: broken pipe +Get http://localhost:6060: net/http: transport closed before response was received +Get http://localhost:6060: http: can't write HTTP request on broken connection ``` + ##### json ```json { @@ -172,14 +177,9 @@ func main() { duration := 4 * time.Second results := vegeta.Attack(targets, rate, duration) + metrics := vegeta.NewMetrics(results) - totalTime := time.Duration(0) - for _, result := range results { - totalTime += result.Timing - } - meanTime := time.Duration(float64(totalTime) / float64(len(results))) - - fmt.Printf("Average timing: %s", meanTime) + fmt.Printf("Mean latency: %s", metrics.Latencies.Mean) } ``` diff --git a/lib/attack.go b/lib/attack.go index 82a9441..31f4cae 100644 --- a/lib/attack.go +++ b/lib/attack.go @@ -56,7 +56,7 @@ func hit(req *http.Request, res chan Result) { r, err := client.Do(req) result := Result{ Timestamp: began, - Timing: time.Since(began), + Latency: time.Since(began), BytesOut: uint64(req.ContentLength), } if err != nil { diff --git a/lib/metrics.go b/lib/metrics.go index 6592156..2feb786 100644 --- a/lib/metrics.go +++ b/lib/metrics.go @@ -3,49 +3,70 @@ package vegeta import ( "strconv" "time" + + "github.com/bmizerany/perks/quantile" ) // Metrics holds the stats computed out of a slice of Results // that is used for some of the Reporters type Metrics struct { - TotalRequests uint64 `json:"total_requests"` - TotalTiming time.Duration `json:"total_timing"` - MeanTiming time.Duration `json:"mean_timing"` - TotalBytesIn uint64 `json:"total_bytes_in"` - MeanBytesIn float64 `json:"mean_bytes_in"` - TotalBytesOut uint64 `json:"total_bytes_out"` - MeanBytesOut float64 `json:"mean_bytes_out"` - TotalSuccess uint64 `json:"total_success"` - MeanSuccess float64 `json:"mean_success"` - StatusCodes map[string]int `json:"status_codes"` - Errors []string `json:"errors"` + Latencies struct { + Total time.Duration `json:"total"` + Max time.Duration `json:"max"` + Mean time.Duration `json:"mean"` + Mean95 time.Duration `json:"mean_95"` + Mean99 time.Duration `json:"mean_99"` + } `json:"latencies"` + + BytesIn struct { + Total uint64 `json:"total"` + Mean float64 `json:"mean"` + } `json:"bytes_in"` + + BytesOut struct { + Total uint64 `json:"total"` + Mean float64 `json:"mean"` + } `json:"bytes_out"` + + Requests uint64 `json:"requests"` + Success float64 `json:"success"` + StatusCodes map[string]int `json:"status_codes"` + Errors []string `json:"errors"` } // NewMetrics computes and returns a Metrics struct out of a slice of Results func NewMetrics(results []Result) *Metrics { m := &Metrics{ - TotalRequests: uint64(len(results)), - StatusCodes: map[string]int{}, + Requests: uint64(len(results)), + StatusCodes: map[string]int{}, } errorSet := map[string]struct{}{} + quants := quantile.NewTargeted(0.95, 0.99) + totalSuccess := 0 for _, result := range results { + quants.Insert(float64(result.Latency)) m.StatusCodes[strconv.Itoa(int(result.Code))]++ - m.TotalTiming += result.Timing - m.TotalBytesOut += result.BytesOut - m.TotalBytesIn += result.BytesIn + m.Latencies.Total += result.Latency + m.BytesOut.Total += result.BytesOut + m.BytesIn.Total += result.BytesIn + if result.Latency > m.Latencies.Max { + m.Latencies.Max = result.Latency + } if result.Code >= 200 && result.Code < 300 { - m.TotalSuccess++ + totalSuccess++ } if result.Error != "" { errorSet[result.Error] = struct{}{} } } - m.MeanTiming = time.Duration(float64(m.TotalTiming) / float64(m.TotalRequests)) - m.MeanBytesOut = float64(m.TotalBytesOut) / float64(m.TotalRequests) - m.MeanBytesIn = float64(m.TotalBytesIn) / float64(m.TotalRequests) - m.MeanSuccess = float64(m.TotalSuccess) / float64(m.TotalRequests) + m.Latencies.Mean = time.Duration(float64(m.Latencies.Total) / float64(m.Requests)) + m.Latencies.Mean95 = time.Duration(quants.Query(0.95)) + m.Latencies.Mean99 = time.Duration(quants.Query(0.99)) + m.BytesIn.Mean = float64(m.BytesIn.Total) / float64(m.Requests) + m.BytesOut.Mean = float64(m.BytesOut.Total) / float64(m.Requests) + m.Success = float64(totalSuccess) / float64(m.Requests) m.Errors = make([]string, 0, len(errorSet)) for err := range errorSet { diff --git a/lib/metrics_test.go b/lib/metrics_test.go index 94571ab..25e171e 100644 --- a/lib/metrics_test.go +++ b/lib/metrics_test.go @@ -13,9 +13,9 @@ func TestNewMetrics(t *testing.T) { }) for field, values := range map[string][]float64{ - "MeanBytesIn": []float64{m.MeanBytesIn, 20.0}, - "MeanBytesOut": []float64{m.MeanBytesOut, 20.0}, - "MeanSuccess": []float64{m.MeanSuccess, 0.6666666666666666}, + "BytesIn.Mean": []float64{m.BytesIn.Mean, 20.0}, + "BytesOut.Mean": []float64{m.BytesOut.Mean, 20.0}, + "Sucess": []float64{m.Success, 0.6666666666666666}, } { if values[0] != values[1] { t.Errorf("%s: want: %f, got: %f", field, values[1], values[0]) @@ -23,8 +23,11 @@ func TestNewMetrics(t *testing.T) { } for field, values := range map[string][]time.Duration{ - "TotalTiming": []time.Duration{m.TotalTiming, 150 * time.Millisecond}, - "MeanTiming": []time.Duration{m.MeanTiming, 50 * time.Millisecond}, + "Latencies.Total": []time.Duration{m.Latencies.Total, 150 * time.Millisecond}, + "Latencies.Mean": []time.Duration{m.Latencies.Mean, 50 * time.Millisecond}, + "Latencies.Mean95": []time.Duration{m.Latencies.Mean95, 30 * time.Millisecond}, + "Latencies.Mean99": []time.Duration{m.Latencies.Mean99, 30 * time.Millisecond}, + "Latencies.Max": []time.Duration{m.Latencies.Max, 100 * time.Millisecond}, } { if values[0] != values[1] { t.Errorf("%s: want: %s, got: %s", field, values[1], values[0]) @@ -32,10 +35,9 @@ func TestNewMetrics(t *testing.T) { } for field, values := range map[string][]uint64{ - "TotalSuccess": []uint64{m.TotalSuccess, 2}, - "TotalBytesOut": []uint64{m.TotalBytesOut, 60}, - "TotalRequests": []uint64{m.TotalRequests, 3}, - "TotalBytesIn": []uint64{m.TotalBytesIn, 60}, + "BytesOut.Total": []uint64{m.BytesOut.Total, 60}, + "BytesIn.Total": []uint64{m.BytesIn.Total, 60}, + "Requests": []uint64{m.Requests, 3}, } { if values[0] != values[1] { t.Errorf("%s: want: %d, got: %d", field, values[1], values[0]) diff --git a/lib/reporters.go b/lib/reporters.go index cb98250..eb74791 100644 --- a/lib/reporters.go +++ b/lib/reporters.go @@ -18,19 +18,17 @@ func ReportText(results []Result) ([]byte, error) { out := &bytes.Buffer{} w := tabwriter.NewWriter(out, 0, 8, 2, '\t', tabwriter.StripEscape) - fmt.Fprintf(w, "Time(avg)\tRequests\tSuccess\tBytes(rx/tx)\n") - fmt.Fprintf(w, "%s\t%d\t%.2f%%\t%.2f/%.2f\n", m.MeanTiming, m.TotalRequests, m.MeanSuccess*100, m.MeanBytesIn, m.MeanBytesOut) - - fmt.Fprintf(w, "\nCount:\t") - for _, count := range m.StatusCodes { - fmt.Fprintf(w, "%d\t", count) + fmt.Fprintf(w, "Requests\t[total]\t%d\n", m.Requests) + fmt.Fprintf(w, "Latencies\t[mean, max, 95, 99]\t%s, %s, %s, %s\n", + m.Latencies.Mean, m.Latencies.Max, m.Latencies.Mean95, m.Latencies.Mean99) + fmt.Fprintf(w, "Bytes In\t[total, mean]\t%d, %.2f\n", m.BytesIn.Total, m.BytesIn.Mean) + fmt.Fprintf(w, "Bytes Out\t[total, mean]\t%d, %.2f\n", m.BytesOut.Total, m.BytesOut.Mean) + fmt.Fprintf(w, "Success\t[ratio]\t%.2f%%\n", m.Success*100) + fmt.Fprintf(w, "Status Codes\t[code:count]\t") + for code, count := range m.StatusCodes { + fmt.Fprintf(w, "%s:%d ", code, count) } - fmt.Fprintf(w, "\nStatus:\t") - for code := range m.StatusCodes { - fmt.Fprintf(w, "%s\t", code) - } - - fmt.Fprintln(w, "\n\nError Set:") + fmt.Fprintln(w, "\nError Set:") for _, err := range m.Errors { fmt.Fprintln(w, err) } @@ -53,7 +51,7 @@ func ReportPlot(results []Result) ([]byte, error) { for _, result := range results { fmt.Fprintf(out, "[%f,%f],", result.Timestamp.Sub(results[0].Timestamp).Seconds(), - result.Timing.Seconds()*1000, + result.Latency.Seconds()*1000, ) } out.Truncate(out.Len() - 1) // Remove trailing comma diff --git a/lib/results.go b/lib/results.go index 0c7c305..94e4e40 100644 --- a/lib/results.go +++ b/lib/results.go @@ -12,7 +12,7 @@ import ( type Result struct { Code uint16 Timestamp time.Time - Timing time.Duration + Latency time.Duration BytesOut uint64 BytesIn uint64 Error string