From 078d4c861b461d8fb27283a2eb1ced599bfcbbb5 Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Sat, 1 Apr 2023 19:00:50 +0800 Subject: [PATCH] feat: support output the test report (#17) Co-authored-by: rick --- CONTRIBUTION.md | 5 ++ Makefile | 2 +- cmd/run.go | 47 +++++++++++- cmd/run_test.go | 7 +- pkg/runner/data/report.md | 5 ++ pkg/runner/reporter_discard.go | 17 +++++ pkg/runner/reporter_discard_test.go | 20 +++++ pkg/runner/reporter_memory.go | 68 +++++++++++++++++ pkg/runner/simple.go | 112 +++++++++++++++++++++++++++- pkg/runner/writer_std.go | 58 ++++++++++++++ pkg/runner/writer_std_test.go | 80 ++++++++++++++++++++ 11 files changed, 408 insertions(+), 13 deletions(-) create mode 100644 CONTRIBUTION.md create mode 100644 pkg/runner/data/report.md create mode 100644 pkg/runner/reporter_discard.go create mode 100644 pkg/runner/reporter_discard_test.go create mode 100644 pkg/runner/reporter_memory.go create mode 100644 pkg/runner/writer_std.go create mode 100644 pkg/runner/writer_std_test.go diff --git a/CONTRIBUTION.md b/CONTRIBUTION.md new file mode 100644 index 0000000..14ea9a9 --- /dev/null +++ b/CONTRIBUTION.md @@ -0,0 +1,5 @@ +Print the code of lines: + +```shell +git ls-files | xargs cloc +``` diff --git a/Makefile b/Makefile index d3ea59e..0537c82 100644 --- a/Makefile +++ b/Makefile @@ -6,4 +6,4 @@ build: copy: build sudo cp bin/atest /usr/local/bin/ test: - go test ./... + go test ./... -cover diff --git a/cmd/run.go b/cmd/run.go index 7cd9e29..c63c9cf 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "path" "path/filepath" "strings" @@ -28,18 +29,36 @@ type runOption struct { burst int32 limiter limit.RateLimiter startTime time.Time + reporter runner.TestReporter + reportWriter runner.ReportResultWriter + report string +} + +func newDefaultRunOption() *runOption { + return &runOption{ + reporter: runner.NewmemoryTestReporter(), + reportWriter: runner.NewResultWriter(os.Stdout), + } +} + +func newDiskCardRunOption() *runOption { + return &runOption{ + reporter: runner.NewDiscardTestReporter(), + reportWriter: runner.NewDiscardResultWriter(), + } } // CreateRunCommand returns the run command func CreateRunCommand() (cmd *cobra.Command) { - opt := &runOption{} + opt := newDefaultRunOption() cmd = &cobra.Command{ Use: "run", Aliases: []string{"r"}, Example: `atest run -p sample.yaml See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`, - Short: "Run the test suite", - RunE: opt.runE, + Short: "Run the test suite", + PreRunE: opt.preRunE, + RunE: opt.runE, } // set flags @@ -52,6 +71,15 @@ See also https://github.com/LinuxSuRen/api-testing/tree/master/sample`, flags.Int64VarP(&opt.thread, "thread", "", 1, "Threads of the execution") flags.Int32VarP(&opt.qps, "qps", "", 5, "QPS") flags.Int32VarP(&opt.burst, "burst", "", 5, "burst") + flags.StringVarP(&opt.report, "report", "", "", "The type of target report") + return +} + +func (o *runOption) preRunE(cmd *cobra.Command, args []string) (err error) { + switch o.report { + case "markdown", "md": + o.reportWriter = runner.NewMarkdownResultWriter(cmd.OutOrStdout()) + } return } @@ -73,6 +101,14 @@ func (o *runOption) runE(cmd *cobra.Command, args []string) (err error) { } } } + + // print the report + if err == nil { + var results []runner.ReportResult + if results, err = o.reporter.ExportAllReportResults(); err == nil { + err = o.reportWriter.Output(results) + } + } return } @@ -165,7 +201,10 @@ func (o *runOption) runSuite(suite string, dataContext map[string]interface{}, c o.limiter.Accept() ctxWithTimeout, _ := context.WithTimeout(ctx, o.requestTimeout) - if output, err = runner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError { + + simpleRunner := runner.NewSimpleTestCaseRunner() + simpleRunner.WithTestReporter(o.reporter) + if output, err = simpleRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError { return } } diff --git a/cmd/run_test.go b/cmd/run_test.go index 7e73895..8779758 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -49,10 +49,9 @@ func TestRunSuite(t *testing.T) { tt.prepare() ctx := getDefaultContext() - opt := &runOption{ - requestTimeout: 30 * time.Second, - limiter: limit.NewDefaultRateLimiter(0, 0), - } + opt := newDiskCardRunOption() + opt.requestTimeout = 30 * time.Second + opt.limiter = limit.NewDefaultRateLimiter(0, 0) stopSingal := make(chan struct{}, 1) err := opt.runSuite(tt.suiteFile, ctx, context.TODO(), stopSingal) diff --git a/pkg/runner/data/report.md b/pkg/runner/data/report.md new file mode 100644 index 0000000..3317c77 --- /dev/null +++ b/pkg/runner/data/report.md @@ -0,0 +1,5 @@ +| API | Average | Max | Min | Count | Error | +|---|---|---|---|---|---| +{{- range $val := .}} +| {{$val.API}} | {{$val.Average}} | {{$val.Max}} | {{$val.Min}} | {{$val.Count}} | {{$val.Error}} | +{{- end}} diff --git a/pkg/runner/reporter_discard.go b/pkg/runner/reporter_discard.go new file mode 100644 index 0000000..4d1d14e --- /dev/null +++ b/pkg/runner/reporter_discard.go @@ -0,0 +1,17 @@ +package runner + +type discardTestReporter struct { +} + +// NewDiscardTestReporter creates a test reporter which discard everything +func NewDiscardTestReporter() TestReporter { + return &discardTestReporter{} +} + +func (r *discardTestReporter) PutRecord(*ReportRecord) {} +func (r *discardTestReporter) GetAllRecords() []*ReportRecord { + return nil +} +func (r *discardTestReporter) ExportAllReportResults() (ReportResultSlice, error) { + return nil, nil +} diff --git a/pkg/runner/reporter_discard_test.go b/pkg/runner/reporter_discard_test.go new file mode 100644 index 0000000..f5ef26d --- /dev/null +++ b/pkg/runner/reporter_discard_test.go @@ -0,0 +1,20 @@ +package runner_test + +import ( + "testing" + + "github.com/linuxsuren/api-testing/pkg/runner" + "github.com/stretchr/testify/assert" +) + +func TestDiscardTestReporter(t *testing.T) { + reporter := runner.NewDiscardTestReporter() + assert.NotNil(t, reporter) + assert.Nil(t, reporter.GetAllRecords()) + + result, err := reporter.ExportAllReportResults() + assert.Nil(t, result) + assert.Nil(t, err) + + reporter.PutRecord(&runner.ReportRecord{}) +} diff --git a/pkg/runner/reporter_memory.go b/pkg/runner/reporter_memory.go new file mode 100644 index 0000000..150d65c --- /dev/null +++ b/pkg/runner/reporter_memory.go @@ -0,0 +1,68 @@ +package runner + +import ( + "sort" + "time" +) + +type memoryTestReporter struct { + records []*ReportRecord +} + +// NewmemoryTestReporter creates a memory based test reporter +func NewmemoryTestReporter() TestReporter { + return &memoryTestReporter{ + records: []*ReportRecord{}, + } +} + +type ReportResultWithTotal struct { + ReportResult + Total time.Duration +} + +func (r *memoryTestReporter) PutRecord(record *ReportRecord) { + r.records = append(r.records, record) +} +func (r *memoryTestReporter) GetAllRecords() []*ReportRecord { + return r.records +} +func (r *memoryTestReporter) ExportAllReportResults() (result ReportResultSlice, err error) { + resultWithTotal := map[string]*ReportResultWithTotal{} + for _, record := range r.records { + api := record.Method + " " + record.API + duration := record.Duration() + + if item, ok := resultWithTotal[api]; ok { + if item.Max < duration { + item.Max = duration + } + + if item.Min > duration { + item.Min = duration + } + item.Error += record.ErrorCount() + item.Total += duration + item.Count += 1 + } else { + resultWithTotal[api] = &ReportResultWithTotal{ + ReportResult: ReportResult{ + API: api, + Count: 1, + Max: duration, + Min: duration, + Error: record.ErrorCount(), + }, + Total: duration, + } + } + } + + for _, r := range resultWithTotal { + r.Average = r.Total / time.Duration(r.Count) + result = append(result, r.ReportResult) + } + + sort.Sort(result) + return +} diff --git a/pkg/runner/simple.go b/pkg/runner/simple.go index 0407fc4..7dd5a18 100644 --- a/pkg/runner/simple.go +++ b/pkg/runner/simple.go @@ -12,6 +12,7 @@ import ( "os" "reflect" "strings" + "time" "github.com/andreyvit/diff" "github.com/antonmedv/expr" @@ -21,9 +22,94 @@ import ( unstructured "github.com/linuxsuren/unstructured/pkg" ) -// RunTestCase runs the test case -func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) { - fmt.Printf("start to run: '%s'\n", testcase.Name) +type TestCaseRunner interface { + RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) + WithOutputWriter(io.Writer) TestCaseRunner + WithTestReporter(TestReporter) TestCaseRunner +} + +type ReportRecord struct { + Method string + API string + BeginTime time.Time + EndTime time.Time + Error error +} + +// Duration returns the duration between begin and end time +func (r *ReportRecord) Duration() time.Duration { + return r.EndTime.Sub(r.BeginTime) +} + +func (r *ReportRecord) ErrorCount() int { + if r.Error == nil { + return 0 + } + return 1 +} + +// NewReportRecord creates a record, and set the begin time to be now +func NewReportRecord() *ReportRecord { + return &ReportRecord{ + BeginTime: time.Now(), + } +} + +type ReportResult struct { + API string + Count int + Average time.Duration + Max time.Duration + Min time.Duration + Error int +} + +type ReportResultSlice []ReportResult + +func (r ReportResultSlice) Len() int { + return len(r) +} + +func (r ReportResultSlice) Less(i, j int) bool { + return r[i].Average > r[j].Average +} + +func (r ReportResultSlice) Swap(i, j int) { + tmp := r[i] + r[i] = r[j] + r[j] = tmp +} + +type ReportResultWriter interface { + Output([]ReportResult) error +} + +type TestReporter interface { + PutRecord(*ReportRecord) + GetAllRecords() []*ReportRecord + ExportAllReportResults() (ReportResultSlice, error) +} + +type simpleTestCaseRunner struct { + testReporter TestReporter + writer io.Writer +} + +// NewSimpleTestCaseRunner creates the instance of the simple test case runner +func NewSimpleTestCaseRunner() TestCaseRunner { + runner := &simpleTestCaseRunner{} + return runner.WithOutputWriter(io.Discard).WithTestReporter(NewDiscardTestReporter()) +} + +func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) { + fmt.Fprintf(r.writer, "start to run: '%s'\n", testcase.Name) + record := NewReportRecord() + defer func(rr *ReportRecord) { + rr.EndTime = time.Now() + rr.Error = err + r.testReporter.PutRecord(rr) + }(record) + if err = doPrepare(testcase); err != nil { err = fmt.Errorf("failed to prepare, error: %v", err) return @@ -77,13 +163,15 @@ func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx contex if request, err = http.NewRequestWithContext(ctx, testcase.Request.Method, testcase.Request.API, requestBody); err != nil { return } + record.API = testcase.Request.API + record.Method = testcase.Request.Method // set headers for key, val := range testcase.Request.Header { request.Header.Add(key, val) } - fmt.Println("start to send request to", testcase.Request.API) + fmt.Fprintf(r.writer, "start to send request to %s\n", testcase.Request.API) // send the HTTP request var resp *http.Response @@ -178,6 +266,22 @@ func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx contex return } +func (r *simpleTestCaseRunner) WithOutputWriter(writer io.Writer) TestCaseRunner { + r.writer = writer + return r +} + +func (r *simpleTestCaseRunner) WithTestReporter(reporter TestReporter) TestCaseRunner { + r.testReporter = reporter + return r +} + +// Deprecated +// RunTestCase runs the test case. +func RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) { + return NewSimpleTestCaseRunner().WithOutputWriter(os.Stdout).RunTestCase(testcase, dataContext, ctx) +} + func doPrepare(testcase *testing.TestCase) (err error) { for i := range testcase.Prepare.Kubernetes { item := testcase.Prepare.Kubernetes[i] diff --git a/pkg/runner/writer_std.go b/pkg/runner/writer_std.go new file mode 100644 index 0000000..4359f42 --- /dev/null +++ b/pkg/runner/writer_std.go @@ -0,0 +1,58 @@ +package runner + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "os" + "text/template" +) + +type stdResultWriter struct { + writer io.Writer +} + +func NewResultWriter(writer io.Writer) ReportResultWriter { + return &stdResultWriter{writer: writer} +} + +// NewDiscardResultWriter creates a report result writer which discard everything +func NewDiscardResultWriter() ReportResultWriter { + return &stdResultWriter{writer: io.Discard} +} + +func (w *stdResultWriter) Output(result []ReportResult) error { + fmt.Fprintf(w.writer, "API Average Max Min Count Error\n") + for _, r := range result { + fmt.Fprintf(w.writer, "%s %v %v %v %d %d\n", r.API, r.Average, r.Max, + r.Min, r.Count, r.Error) + } + return nil +} + +type markdownResultWriter struct { + writer io.Writer +} + +func NewMarkdownResultWriter(writer io.Writer) ReportResultWriter { + if writer == nil { + writer = os.Stdout + } + return &markdownResultWriter{writer: writer} +} + +func (w *markdownResultWriter) Output(result []ReportResult) (err error) { + var tpl *template.Template + if tpl, err = template.New("report").Parse(markDownReport); err == nil { + buf := new(bytes.Buffer) + + if err = tpl.Execute(buf, result); err == nil { + fmt.Fprint(w.writer, buf.String()) + } + } + return +} + +//go:embed data/report.md +var markDownReport string diff --git a/pkg/runner/writer_std_test.go b/pkg/runner/writer_std_test.go new file mode 100644 index 0000000..9bc5812 --- /dev/null +++ b/pkg/runner/writer_std_test.go @@ -0,0 +1,80 @@ +package runner_test + +import ( + "bytes" + "testing" + + "github.com/linuxsuren/api-testing/pkg/runner" + "github.com/stretchr/testify/assert" +) + +func TestMarkdownWriter(t *testing.T) { + buf := new(bytes.Buffer) + writer := runner.NewMarkdownResultWriter(buf) + + err := writer.Output([]runner.ReportResult{{ + API: "api", + Average: 3, + Max: 4, + Min: 2, + Count: 3, + Error: 0, + }, { + API: "api", + Average: 3, + Max: 4, + Min: 2, + Count: 3, + Error: 0, + }}) + assert.Nil(t, err) + assert.Equal(t, `| API | Average | Max | Min | Count | Error | +|---|---|---|---|---|---| +| api | 3ns | 4ns | 2ns | 3 | 0 | +| api | 3ns | 4ns | 2ns | 3 | 0 | +`, buf.String()) +} + +func TestNewStdResultWriter(t *testing.T) { + tests := []struct { + name string + buf *bytes.Buffer + results []runner.ReportResult + expect string + }{{ + name: "result is nil", + buf: new(bytes.Buffer), + results: nil, + expect: `API Average Max Min Count Error +`, + }, { + name: "have one item", + buf: new(bytes.Buffer), + results: []runner.ReportResult{{ + API: "api", + Average: 1, + Max: 1, + Min: 1, + Count: 1, + Error: 0, + }}, + expect: `API Average Max Min Count Error +api 1ns 1ns 1ns 1 0 +`, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + writer := runner.NewResultWriter(tt.buf) + if !assert.NotNil(t, writer) { + return + } + + err := writer.Output(tt.results) + assert.Nil(t, err) + assert.Equal(t, tt.expect, tt.buf.String()) + }) + } + + discardResultWriter := runner.NewDiscardResultWriter() + assert.NotNil(t, discardResultWriter) +}