Extend Metrics with max and 95th, 99th percentiles

Implements a new format for the text reporter with more information
than before.
Also Rename Result.Timing to Result.Latency and related names.
Adapt README.md examples to account new text reporter format and
library changes.

lib/attack.go

@@ -56,7 +56,7 @@ func hit(req *http.Request, res chan Result) {
 	r, err := client.Do(req)
 	result := Result{
 		Timestamp: began,
-		Timing:    time.Since(began),
+		Latency:   time.Since(began),
 		BytesOut:  uint64(req.ContentLength),
 	}
 	if err != nil {

lib/metrics.go

@@ -3,49 +3,70 @@ package vegeta
 import (
 	"strconv"
 	"time"
+
+	"github.com/bmizerany/perks/quantile"
 )
 
 // Metrics holds the stats computed out of a slice of Results
 // that is used for some of the Reporters
 type Metrics struct {
-	TotalRequests uint64         `json:"total_requests"`
-	TotalTiming   time.Duration  `json:"total_timing"`
-	MeanTiming    time.Duration  `json:"mean_timing"`
-	TotalBytesIn  uint64         `json:"total_bytes_in"`
-	MeanBytesIn   float64        `json:"mean_bytes_in"`
-	TotalBytesOut uint64         `json:"total_bytes_out"`
-	MeanBytesOut  float64        `json:"mean_bytes_out"`
-	TotalSuccess  uint64         `json:"total_success"`
-	MeanSuccess   float64        `json:"mean_success"`
-	StatusCodes   map[string]int `json:"status_codes"`
-	Errors        []string       `json:"errors"`
+	Latencies struct {
+		Total  time.Duration `json:"total"`
+		Max    time.Duration `json:"max"`
+		Mean   time.Duration `json:"mean"`
+		Mean95 time.Duration `json:"mean_95"`
+		Mean99 time.Duration `json:"mean_99"`
+	} `json:"latencies"`
+
+	BytesIn struct {
+		Total uint64  `json:"total"`
+		Mean  float64 `json:"mean"`
+	} `json:"bytes_in"`
+
+	BytesOut struct {
+		Total uint64  `json:"total"`
+		Mean  float64 `json:"mean"`
+	} `json:"bytes_out"`
+
+	Requests    uint64         `json:"requests"`
+	Success     float64        `json:"success"`
+	StatusCodes map[string]int `json:"status_codes"`
+	Errors      []string       `json:"errors"`
 }
 
 // NewMetrics computes and returns a Metrics struct out of a slice of Results
 func NewMetrics(results []Result) *Metrics {
 	m := &Metrics{
-		TotalRequests: uint64(len(results)),
-		StatusCodes:   map[string]int{},
+		Requests:    uint64(len(results)),
+		StatusCodes: map[string]int{},
 	}
 	errorSet := map[string]struct{}{}
+	quants := quantile.NewTargeted(0.95, 0.99)
+	totalSuccess := 0
 
 	for _, result := range results {
+		quants.Insert(float64(result.Latency))
 		m.StatusCodes[strconv.Itoa(int(result.Code))]++
-		m.TotalTiming += result.Timing
-		m.TotalBytesOut += result.BytesOut
-		m.TotalBytesIn += result.BytesIn
+		m.Latencies.Total += result.Latency
+		m.BytesOut.Total += result.BytesOut
+		m.BytesIn.Total += result.BytesIn
+		if result.Latency > m.Latencies.Max {
+			m.Latencies.Max = result.Latency
+		}
 		if result.Code >= 200 && result.Code < 300 {
-			m.TotalSuccess++
+			totalSuccess++
 		}
 		if result.Error != "" {
 			errorSet[result.Error] = struct{}{}
 		}
 	}
 
-	m.MeanTiming = time.Duration(float64(m.TotalTiming) / float64(m.TotalRequests))
-	m.MeanBytesOut = float64(m.TotalBytesOut) / float64(m.TotalRequests)
-	m.MeanBytesIn = float64(m.TotalBytesIn) / float64(m.TotalRequests)
-	m.MeanSuccess = float64(m.TotalSuccess) / float64(m.TotalRequests)
+	m.Latencies.Mean = time.Duration(float64(m.Latencies.Total) / float64(m.Requests))
+	m.Latencies.Mean95 = time.Duration(quants.Query(0.95))
+	m.Latencies.Mean99 = time.Duration(quants.Query(0.99))
+	m.BytesIn.Mean = float64(m.BytesIn.Total) / float64(m.Requests)
+	m.BytesOut.Mean = float64(m.BytesOut.Total) / float64(m.Requests)
+	m.Success = float64(totalSuccess) / float64(m.Requests)
 
 	m.Errors = make([]string, 0, len(errorSet))
 	for err := range errorSet {

lib/metrics_test.go

@@ -13,9 +13,9 @@ func TestNewMetrics(t *testing.T) {
 	})
 
 	for field, values := range map[string][]float64{
-		"MeanBytesIn":  []float64{m.MeanBytesIn, 20.0},
-		"MeanBytesOut": []float64{m.MeanBytesOut, 20.0},
-		"MeanSuccess":  []float64{m.MeanSuccess, 0.6666666666666666},
+		"BytesIn.Mean":  []float64{m.BytesIn.Mean, 20.0},
+		"BytesOut.Mean": []float64{m.BytesOut.Mean, 20.0},
+		"Sucess":        []float64{m.Success, 0.6666666666666666},
 	} {
 		if values[0] != values[1] {
 			t.Errorf("%s: want: %f, got: %f", field, values[1], values[0])
@@ -23,8 +23,11 @@ func TestNewMetrics(t *testing.T) {
 	}
 
 	for field, values := range map[string][]time.Duration{
-		"TotalTiming": []time.Duration{m.TotalTiming, 150 * time.Millisecond},
-		"MeanTiming":  []time.Duration{m.MeanTiming, 50 * time.Millisecond},
+		"Latencies.Total":  []time.Duration{m.Latencies.Total, 150 * time.Millisecond},
+		"Latencies.Mean":   []time.Duration{m.Latencies.Mean, 50 * time.Millisecond},
+		"Latencies.Mean95": []time.Duration{m.Latencies.Mean95, 30 * time.Millisecond},
+		"Latencies.Mean99": []time.Duration{m.Latencies.Mean99, 30 * time.Millisecond},
+		"Latencies.Max":    []time.Duration{m.Latencies.Max, 100 * time.Millisecond},
 	} {
 		if values[0] != values[1] {
 			t.Errorf("%s: want: %s, got: %s", field, values[1], values[0])
@@ -32,10 +35,9 @@ func TestNewMetrics(t *testing.T) {
 	}
 
 	for field, values := range map[string][]uint64{
-		"TotalSuccess":  []uint64{m.TotalSuccess, 2},
-		"TotalBytesOut": []uint64{m.TotalBytesOut, 60},
-		"TotalRequests": []uint64{m.TotalRequests, 3},
-		"TotalBytesIn":  []uint64{m.TotalBytesIn, 60},
+		"BytesOut.Total": []uint64{m.BytesOut.Total, 60},
+		"BytesIn.Total":  []uint64{m.BytesIn.Total, 60},
+		"Requests":       []uint64{m.Requests, 3},
 	} {
 		if values[0] != values[1] {
 			t.Errorf("%s: want: %d, got: %d", field, values[1], values[0])

lib/reporters.go

@@ -18,19 +18,17 @@ func ReportText(results []Result) ([]byte, error) {
 	out := &bytes.Buffer{}
 
 	w := tabwriter.NewWriter(out, 0, 8, 2, '\t', tabwriter.StripEscape)
-	fmt.Fprintf(w, "Time(avg)\tRequests\tSuccess\tBytes(rx/tx)\n")
-	fmt.Fprintf(w, "%s\t%d\t%.2f%%\t%.2f/%.2f\n", m.MeanTiming, m.TotalRequests, m.MeanSuccess*100, m.MeanBytesIn, m.MeanBytesOut)
-
-	fmt.Fprintf(w, "\nCount:\t")
-	for _, count := range m.StatusCodes {
-		fmt.Fprintf(w, "%d\t", count)
+	fmt.Fprintf(w, "Requests\t[total]\t%d\n", m.Requests)
+	fmt.Fprintf(w, "Latencies\t[mean, max, 95, 99]\t%s, %s, %s, %s\n",
+		m.Latencies.Mean, m.Latencies.Max, m.Latencies.Mean95, m.Latencies.Mean99)
+	fmt.Fprintf(w, "Bytes In\t[total, mean]\t%d, %.2f\n", m.BytesIn.Total, m.BytesIn.Mean)
+	fmt.Fprintf(w, "Bytes Out\t[total, mean]\t%d, %.2f\n", m.BytesOut.Total, m.BytesOut.Mean)
+	fmt.Fprintf(w, "Success\t[ratio]\t%.2f%%\n", m.Success*100)
+	fmt.Fprintf(w, "Status Codes\t[code:count]\t")
+	for code, count := range m.StatusCodes {
+		fmt.Fprintf(w, "%s:%d  ", code, count)
 	}
-	fmt.Fprintf(w, "\nStatus:\t")
-	for code := range m.StatusCodes {
-		fmt.Fprintf(w, "%s\t", code)
-	}
-
-	fmt.Fprintln(w, "\n\nError Set:")
+	fmt.Fprintln(w, "\nError Set:")
 	for _, err := range m.Errors {
 		fmt.Fprintln(w, err)
 	}
@@ -53,7 +51,7 @@ func ReportPlot(results []Result) ([]byte, error) {
 	for _, result := range results {
 		fmt.Fprintf(out, "[%f,%f],",
 			result.Timestamp.Sub(results[0].Timestamp).Seconds(),
-			result.Timing.Seconds()*1000,
+			result.Latency.Seconds()*1000,
 		)
 	}
 	out.Truncate(out.Len() - 1) // Remove trailing comma

lib/results.go

@@ -12,7 +12,7 @@ import (
 type Result struct {
 	Code      uint16
 	Timestamp time.Time
-	Timing    time.Duration
+	Latency   time.Duration
 	BytesOut  uint64
 	BytesIn   uint64
 	Error     string