feat: support output the test report (#17)

Co-authored-by: rick <LinuxSuRen@users.noreply.github.com>

CONTRIBUTION.md

@@ -0,0 +1,5 @@
+Print the code of lines:
+
+```shell
+git ls-files | xargs cloc
+```

Makefile

@@ -6,4 +6,4 @@ build:
 copy: build
 	sudo cp bin/atest /usr/local/bin/
 test:
-	go test ./...
+	go test ./... -cover

cmd/run.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -28,18 +29,36 @@ type runOption struct {
 	burst              int32
 	limiter            limit.RateLimiter
 	startTime          time.Time
+	reporter           runner.TestReporter
+	reportWriter       runner.ReportResultWriter
+	report             string
+}
+
+func newDefaultRunOption() *runOption {
+	return &runOption{
+		reporter:     runner.NewmemoryTestReporter(),
+		reportWriter: runner.NewResultWriter(os.Stdout),
+	}
+}
+
+func newDiskCardRunOption() *runOption {
+	return &runOption{
+		reporter:     runner.NewDiscardTestReporter(),
+		reportWriter: runner.NewDiscardResultWriter(),
+	}
 }
 
 // CreateRunCommand returns the run command
 func CreateRunCommand() (cmd *cobra.Command) {
-	opt := &runOption{}
+	opt := newDefaultRunOption()
 	cmd = &cobra.Command{
 		Use:     "run",
 		Aliases: []string{"r"},
 		Example: `atest run -p sample.yaml
 See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`,
-		Short: "Run the test suite",
+		Short:   "Run the test suite",
-		RunE:  opt.runE,
+		PreRunE: opt.preRunE,
+		RunE:    opt.runE,
 	}
 
 	// set flags
@@ -52,6 +71,15 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`,
 	flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution")
 	flags.Int32VarP(&opt.qps, "qps", "", 5, "QPS")
 	flags.Int32VarP(&opt.burst, "burst", "", 5, "burst")
+	flags.StringVarP(&opt.report, "report", "", "", "The type of target report")
+	return
+}
+
+func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) {
+	switch o.report {
+	case "markdown", "md":
+		o.reportWriter = runner.NewMarkdownResultWriter(cmd.OutOrStdout())
+	}
 	return
 }
 
@@ -73,6 +101,14 @@ func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) {
 			}
 		}
 	}
+
+	// print the report
+	if err == nil {
+		var results []runner.ReportResult
+		if results, err = o.reporter.ExportAllReportResults(); err == nil {
+			err = o.reportWriter.Output(results)
+		}
+	}
 	return
 }
 
@@ -165,7 +201,10 @@ func (o *runOption) runSuite(suite string, dataContext map[string]interface{}, c
 			o.limiter.Accept()
 
 			ctxWithTimeout, _ := context.WithTimeout(ctx, o.requestTimeout)
-			if output, err = runner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError {
+
+			simpleRunner := runner.NewSimpleTestCaseRunner()
+			simpleRunner.WithTestReporter(o.reporter)
+			if output, err = simpleRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError {
 				return
 			}
 		}

cmd/run_test.go

@@ -49,10 +49,9 @@ func TestRunSuite(t *testing.T) {
 
 			tt.prepare()
 			ctx := getDefaultContext()
-			opt := &runOption{
+			opt := newDiskCardRunOption()
-				requestTimeout: 30 * time.Second,
+			opt.requestTimeout = 30 * time.Second
-				limiter:        limit.NewDefaultRateLimiter(0, 0),
+			opt.limiter = limit.NewDefaultRateLimiter(0, 0)
-			}
 			stopSingal := make(chan struct{}, 1)
 
 			err := opt.runSuite(tt.suiteFile, ctx, context.TODO(), stopSingal)

pkg/runner/data/report.md

@@ -0,0 +1,5 @@
+| API | Average | Max | Min | Count | Error |
+|---|---|---|---|---|---|
+{{- range $val := .}}
+| {{$val.API}} | {{$val.Average}} | {{$val.Max}} | {{$val.Min}} | {{$val.Count}} | {{$val.Error}} |
+{{- end}}

pkg/runner/reporter_discard.go

@@ -0,0 +1,17 @@
+package runner
+
+type discardTestReporter struct {
+}
+
+// NewDiscardTestReporter creates a test reporter which discard everything
+func NewDiscardTestReporter() TestReporter {
+	return &discardTestReporter{}
+}
+
+func (r *discardTestReporter) PutRecord(*ReportRecord) {}
+func (r *discardTestReporter) GetAllRecords() []*ReportRecord {
+	return nil
+}
+func (r *discardTestReporter) ExportAllReportResults() (ReportResultSlice, error) {
+	return nil, nil
+}

pkg/runner/reporter_discard_test.go

@@ -0,0 +1,20 @@
+package runner_test
+
+import (
+	"testing"
+
+	"github.com/linuxsuren/api-testing/pkg/runner"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiscardTestReporter(t *testing.T) {
+	reporter := runner.NewDiscardTestReporter()
+	assert.NotNil(t, reporter)
+	assert.Nil(t, reporter.GetAllRecords())
+
+	result, err := reporter.ExportAllReportResults()
+	assert.Nil(t, result)
+	assert.Nil(t, err)
+
+	reporter.PutRecord(&runner.ReportRecord{})
+}

pkg/runner/reporter_memory.go

@@ -0,0 +1,68 @@
+package runner
+
+import (
+	"sort"
+	"time"
+)
+
+type memoryTestReporter struct {
+	records []*ReportRecord
+}
+
+// NewmemoryTestReporter creates a memory based test reporter
+func NewmemoryTestReporter() TestReporter {
+	return &memoryTestReporter{
+		records: []*ReportRecord{},
+	}
+}
+
+type ReportResultWithTotal struct {
+	ReportResult
+	Total time.Duration
+}
+
+func (r *memoryTestReporter) PutRecord(record *ReportRecord) {
+	r.records = append(r.records, record)
+}
+func (r *memoryTestReporter) GetAllRecords() []*ReportRecord {
+	return r.records
+}
+func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice, err error) {
+	resultWithTotal := map[string]*ReportResultWithTotal{}
+	for _, record := range r.records {
+		api := record.Method + " " + record.API
+		duration := record.Duration()
+
+		if item, ok := resultWithTotal[api]; ok {
+			if item.Max < duration {
+				item.Max = duration
+			}
+
+			if item.Min > duration {
+				item.Min = duration
+			}
+			item.Error += record.ErrorCount()
+			item.Total += duration
+			item.Count += 1
+		} else {
+			resultWithTotal[api] = &ReportResultWithTotal{
+				ReportResult: ReportResult{
+					API:   api,
+					Count: 1,
+					Max:   duration,
+					Min:   duration,
+					Error: record.ErrorCount(),
+				},
+				Total: duration,
+			}
+		}
+	}
+
+	for _, r := range resultWithTotal {
+		r.Average = r.Total / time.Duration(r.Count)
+		result = append(result, r.ReportResult)
+	}
+
+	sort.Sort(result)
+	return
+}

pkg/runner/simple.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/andreyvit/diff"
 	"github.com/antonmedv/expr"
@@ -21,9 +22,94 @@ import (
 	unstructured "github.com/linuxsuren/unstructured/pkg"
 )
 
-// RunTestCase runs the test case
+type TestCaseRunner interface {
-func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) {
+	RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error)
-	fmt.Printf("start to run: '%s'\n", testcase.Name)
+	WithOutputWriter(io.Writer) TestCaseRunner
+	WithTestReporter(TestReporter) TestCaseRunner
+}
+
+type ReportRecord struct {
+	Method    string
+	API       string
+	BeginTime time.Time
+	EndTime   time.Time
+	Error     error
+}
+
+// Duration returns the duration between begin and end time
+func (r *ReportRecord) Duration() time.Duration {
+	return r.EndTime.Sub(r.BeginTime)
+}
+
+func (r *ReportRecord) ErrorCount() int {
+	if r.Error == nil {
+		return 0
+	}
+	return 1
+}
+
+// NewReportRecord creates a record, and set the begin time to be now
+func NewReportRecord() *ReportRecord {
+	return &ReportRecord{
+		BeginTime: time.Now(),
+	}
+}
+
+type ReportResult struct {
+	API     string
+	Count   int
+	Average time.Duration
+	Max     time.Duration
+	Min     time.Duration
+	Error   int
+}
+
+type ReportResultSlice []ReportResult
+
+func (r ReportResultSlice) Len() int {
+	return len(r)
+}
+
+func (r ReportResultSlice) Less(i, j int) bool {
+	return r[i].Average > r[j].Average
+}
+
+func (r ReportResultSlice) Swap(i, j int) {
+	tmp := r[i]
+	r[i] = r[j]
+	r[j] = tmp
+}
+
+type ReportResultWriter interface {
+	Output([]ReportResult) error
+}
+
+type TestReporter interface {
+	PutRecord(*ReportRecord)
+	GetAllRecords() []*ReportRecord
+	ExportAllReportResults() (ReportResultSlice, error)
+}
+
+type simpleTestCaseRunner struct {
+	testReporter TestReporter
+	writer       io.Writer
+}
+
+// NewSimpleTestCaseRunner creates the instance of the simple test case runner
+func NewSimpleTestCaseRunner() TestCaseRunner {
+	runner := &simpleTestCaseRunner{}
+	return runner.WithOutputWriter(io.Discard).WithTestReporter(NewDiscardTestReporter())
+}
+
+func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) {
+	fmt.Fprintf(r.writer, "start to run: '%s'\n", testcase.Name)
+	record := NewReportRecord()
+	defer func(rr *ReportRecord) {
+		rr.EndTime = time.Now()
+		rr.Error = err
+		r.testReporter.PutRecord(rr)
+	}(record)
+
 	if err = doPrepare(testcase); err != nil {
 		err = fmt.Errorf("failed to prepare, error: %v", err)
 		return
@@ -77,13 +163,15 @@ func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx contex
 	if request, err = http.NewRequestWithContext(ctx, testcase.Request.Method, testcase.Request.API, requestBody); err != nil {
 		return
 	}
+	record.API = testcase.Request.API
+	record.Method = testcase.Request.Method
 
 	// set headers
 	for key, val := range testcase.Request.Header {
 		request.Header.Add(key, val)
 	}
 
-	fmt.Println("start to send request to", testcase.Request.API)
+	fmt.Fprintf(r.writer, "start to send request to %s\n", testcase.Request.API)
 
 	// send the HTTP request
 	var resp *http.Response
@@ -178,6 +266,22 @@ func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx contex
 	return
 }
 
+func (r *simpleTestCaseRunner) WithOutputWriter(writer io.Writer) TestCaseRunner {
+	r.writer = writer
+	return r
+}
+
+func (r *simpleTestCaseRunner) WithTestReporter(reporter TestReporter) TestCaseRunner {
+	r.testReporter = reporter
+	return r
+}
+
+// Deprecated
+// RunTestCase runs the test case.
+func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) {
+	return NewSimpleTestCaseRunner().WithOutputWriter(os.Stdout).RunTestCase(testcase, dataContext, ctx)
+}
+
 func doPrepare(testcase *testing.TestCase) (err error) {
 	for i := range testcase.Prepare.Kubernetes {
 		item := testcase.Prepare.Kubernetes[i]

pkg/runner/writer_std.go

@@ -0,0 +1,58 @@
+package runner
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+	"os"
+	"text/template"
+)
+
+type stdResultWriter struct {
+	writer io.Writer
+}
+
+func NewResultWriter(writer io.Writer) ReportResultWriter {
+	return &stdResultWriter{writer: writer}
+}
+
+// NewDiscardResultWriter creates a report result writer which discard everything
+func NewDiscardResultWriter() ReportResultWriter {
+	return &stdResultWriter{writer: io.Discard}
+}
+
+func (w *stdResultWriter) Output(result []ReportResult) error {
+	fmt.Fprintf(w.writer, "API Average Max Min Count Error\n")
+	for _, r := range result {
+		fmt.Fprintf(w.writer, "%s %v %v %v %d %d\n", r.API, r.Average, r.Max,
+			r.Min, r.Count, r.Error)
+	}
+	return nil
+}
+
+type markdownResultWriter struct {
+	writer io.Writer
+}
+
+func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter {
+	if writer == nil {
+		writer = os.Stdout
+	}
+	return &markdownResultWriter{writer: writer}
+}
+
+func (w *markdownResultWriter) Output(result []ReportResult) (err error) {
+	var tpl *template.Template
+	if tpl, err = template.New("report").Parse(markDownReport); err == nil {
+		buf := new(bytes.Buffer)
+
+		if err = tpl.Execute(buf, result); err == nil {
+			fmt.Fprint(w.writer, buf.String())
+		}
+	}
+	return
+}
+
+//go:embed data/report.md
+var markDownReport string

pkg/runner/writer_std_test.go

@@ -0,0 +1,80 @@
+package runner_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/linuxsuren/api-testing/pkg/runner"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarkdownWriter(t *testing.T) {
+	buf := new(bytes.Buffer)
+	writer := runner.NewMarkdownResultWriter(buf)
+
+	err := writer.Output([]runner.ReportResult{{
+		API:     "api",
+		Average: 3,
+		Max:     4,
+		Min:     2,
+		Count:   3,
+		Error:   0,
+	}, {
+		API:     "api",
+		Average: 3,
+		Max:     4,
+		Min:     2,
+		Count:   3,
+		Error:   0,
+	}})
+	assert.Nil(t, err)
+	assert.Equal(t, `| API | Average | Max | Min | Count | Error |
+|---|---|---|---|---|---|
+| api | 3ns | 4ns | 2ns | 3 | 0 |
+| api | 3ns | 4ns | 2ns | 3 | 0 |
+`, buf.String())
+}
+
+func TestNewStdResultWriter(t *testing.T) {
+	tests := []struct {
+		name    string
+		buf     *bytes.Buffer
+		results []runner.ReportResult
+		expect  string
+	}{{
+		name:    "result is nil",
+		buf:     new(bytes.Buffer),
+		results: nil,
+		expect: `API Average Max Min Count Error
+`,
+	}, {
+		name: "have one item",
+		buf:  new(bytes.Buffer),
+		results: []runner.ReportResult{{
+			API:     "api",
+			Average: 1,
+			Max:     1,
+			Min:     1,
+			Count:   1,
+			Error:   0,
+		}},
+		expect: `API Average Max Min Count Error
+api 1ns 1ns 1ns 1 0
+`,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			writer := runner.NewResultWriter(tt.buf)
+			if !assert.NotNil(t, writer) {
+				return
+			}
+
+			err := writer.Output(tt.results)
+			assert.Nil(t, err)
+			assert.Equal(t, tt.expect, tt.buf.String())
+		})
+	}
+
+	discardResultWriter := runner.NewDiscardResultWriter()
+	assert.NotNil(t, discardResultWriter)
+}