From 18bd1987452e2f094e38368a190294e631b46d06 Mon Sep 17 00:00:00 2001 From: Ink33 Date: Sun, 13 Aug 2023 21:09:45 +0800 Subject: [PATCH] feat: support running gRPC test cases (#160) --- cmd/run.go | 6 +- docs/api-testing-schema.json | 41 ++++- go.mod | 7 +- go.sum | 15 +- go.work.sum | 4 + pkg/compare/compare.go | 86 +++++++++ pkg/compare/compare_test.go | 58 +++++++ pkg/compare/error.go | 140 +++++++++++++++ pkg/compare/error_test.go | 35 ++++ pkg/runner/grpc.go | 328 +++++++++++++++++++++++++++++++++++ pkg/runner/grpc_test.go | 23 +++ pkg/runner/http.go | 63 +------ pkg/runner/http_test.go | 3 +- pkg/runner/reporter.go | 2 +- pkg/runner/runner.go | 57 +++++- pkg/runner/runner_factory.go | 32 ++++ pkg/runner/verify.go | 54 ++++++ pkg/server/remote_server.go | 12 +- pkg/testing/case.go | 11 +- sample/grpc-sample.yaml | 47 +++++ sample/halo.yaml | 30 ++-- sample/kubernetes.yaml | 158 ++++++++--------- 22 files changed, 1030 insertions(+), 182 deletions(-) create mode 100644 pkg/compare/compare.go create mode 100644 pkg/compare/compare_test.go create mode 100644 pkg/compare/error.go create mode 100644 pkg/compare/error_test.go create mode 100644 pkg/runner/grpc.go create mode 100644 pkg/runner/grpc_test.go create mode 100644 pkg/runner/runner_factory.go create mode 100644 pkg/runner/verify.go create mode 100644 sample/grpc-sample.yaml diff --git a/cmd/run.go b/cmd/run.go index 62f0098..25c782d 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -254,9 +254,9 @@ func (o *runOption) runSuite(loader testing.Loader, dataContext map[string]inter ctxWithTimeout, _ := context.WithTimeout(ctx, o.requestTimeout) ctxWithTimeout = context.WithValue(ctxWithTimeout, runner.ContextKey("").ParentDir(), loader.GetContext()) - simpleRunner := runner.NewSimpleTestCaseRunner() - simpleRunner.WithTestReporter(o.reporter) - if output, err = simpleRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError { + runner := runner.GetTestSuiteRunner(testSuite) + runner.WithTestReporter(o.reporter) + if output, err = runner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError { err = fmt.Errorf("failed to run '%s', %v", testCase.Name, err) return } else { diff --git a/docs/api-testing-schema.json b/docs/api-testing-schema.json index dff033d..7adb74b 100644 --- a/docs/api-testing-schema.json +++ b/docs/api-testing-schema.json @@ -45,13 +45,38 @@ "properties": { "kind": { "type": "string", - "enum": ["openapi", "swagger"] + "enum": [ + "openapi", + "swagger" + ] }, "url": { "type": "string", "qt-uri-protocols": [ - "https", "http" + "https", + "http" ] + }, + "grpc": { + "type": "object", + "additionalProperties": false, + "properties": { + "import": { + "type": "array", + "items": { + "type": "string" + } + }, + "protofile": { + "type": "string" + }, + "protoset": { + "type": "string" + }, + "serverReflection": { + "type": "boolean" + } + } } } }, @@ -123,10 +148,16 @@ "qt-uri-protocols": [ "https" ] - }, + }, "method": { "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"] + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE" + ] }, "query": { "description": "HTTP request query", @@ -175,4 +206,4 @@ "title": "Job" } } -} +} \ No newline at end of file diff --git a/go.mod b/go.mod index 4be47b6..5d98d1f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/antonmedv/expr v1.12.1 + github.com/bufbuild/protocompile v0.6.0 github.com/cucumber/godog v0.12.6 github.com/ghodss/yaml v1.0.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 @@ -14,12 +15,12 @@ require ( github.com/linuxsuren/go-fake-runtime v0.0.1 github.com/linuxsuren/unstructured v0.0.1 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.14.4 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.3.0 google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index f5340c8..5823918 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antonmedv/expr v1.12.1 h1:GTGrGN1kxxb+le0uQKaFRK8By4cvq1sleUCGE/U6hHg= github.com/antonmedv/expr v1.12.1/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= +github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= +github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -114,7 +116,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -123,8 +124,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -153,8 +154,8 @@ golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -187,8 +188,8 @@ google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/go.work.sum b/go.work.sum index efd37f7..1da4f40 100644 --- a/go.work.sum +++ b/go.work.sum @@ -667,6 +667,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/linuxsuren/go-fake-runtime v0.0.0-20230426144714-1a7a0d160d3f h1:TfAzkLxq/agwMBbccTx/f/dlmFWIBLWRGCWjI4IOlK8= +github.com/linuxsuren/go-fake-runtime v0.0.0-20230426144714-1a7a0d160d3f/go.mod h1:zmh6J78hSnWZo68faMA2eKOdaEp8eFbERHi3ZB9xHCQ= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= @@ -806,6 +808,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/compare/compare.go b/pkg/compare/compare.go new file mode 100644 index 0000000..ce9ac52 --- /dev/null +++ b/pkg/compare/compare.go @@ -0,0 +1,86 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +// Package compare provides basic functions to compare JSON object and array. +// Currently we use gjson to store the JSON data. Please check tidwall/gjson. +package compare + +import ( + "fmt" + "reflect" + "strconv" + + "github.com/tidwall/gjson" +) + +// Object compares two JSON object. +func Object(field string, expect, actul map[string]gjson.Result) error { + var errs error + for k, ev := range expect { + av, ok := actul[k] + if !ok { + errs = JoinErr(errs, newNoEqualErr(field, fmt.Errorf("field %s is not exist", k))) + continue + } + + err := Element(k, ev, av) + if err != nil { + errs = JoinErr(errs, newNoEqualErr(field, err)) + } + } + return errs +} + +// Array compares two JSON Array. +func Array(field string, expect, actul []gjson.Result) error { + var errs error + if l1, l2 := len(expect), len(actul); l1 != l2 { + return newNoEqualErr(field, fmt.Errorf("length is not equal, expect %v fields but got %v", l1, l2)) + } + + for i := range expect { + err := Element(strconv.Itoa(i), expect[i], actul[i]) + if err != nil { + errs = JoinErr(errs, newNoEqualErr(field, err)) + } + } + return errs +} + +// Element compares two JSON Element. +func Element(field string, expect, actul gjson.Result) error { + if expect.Type != actul.Type { + return newNoEqualErr(field, fmt.Errorf("expect type %s but got %v", expect.Type.String(), actul.Type.String())) + } + + if expect.IsObject() { + return Object(field, expect.Map(), actul.Map()) + } + + if expect.IsArray() { + return Array(field, expect.Array(), actul.Array()) + } + + if !reflect.DeepEqual(expect.Value(), actul.Value()) { + return newNoEqualErr(field, fmt.Errorf("expect %v but got %v", expect.Value(), actul.Value())) + } + + return nil +} diff --git a/pkg/compare/compare_test.go b/pkg/compare/compare_test.go new file mode 100644 index 0000000..9422855 --- /dev/null +++ b/pkg/compare/compare_test.go @@ -0,0 +1,58 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package compare + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" +) + +func TestElement(t *testing.T) { + exp := `{ + "data": [ + { + "key": "hell", + "value": "func() strin" + } + ] + } + ` + act := ` + { + "data": [ + { + "key": "hello", + "value": "func() string" + } + ] + }` + expect := gjson.Parse(exp) + actul := gjson.Parse(act) + + err := Element("TestElement", expect, actul) + + expmsg1 := "compare: field TestElement.data.0.value: expect func() strin but got func() string" + expmsg2 := "compare: field TestElement.data.0.key: expect hell but got hello" + assert.Contains(t, err.Error(), expmsg1) + assert.Contains(t, err.Error(), expmsg2) +} diff --git a/pkg/compare/error.go b/pkg/compare/error.go new file mode 100644 index 0000000..168c1bd --- /dev/null +++ b/pkg/compare/error.go @@ -0,0 +1,140 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package compare + +import ( + "fmt" + "strings" +) + +type noEqualErr struct { + Path []string + Message string + NeqErrs *noEqualErrs +} + +type noEqualErrs struct { + errs []error +} + +type task struct { + path []string + errs []error +} + +func (e *noEqualErr) Error() string { + if e.NeqErrs == nil { + return fmt.Sprintf("compare: field %s: %s", strings.Join(e.Path, "."), e.Message) + } + + msg := []string{} + q := []*task{{ + path: e.Path, + errs: e.NeqErrs.errs, + }} + iter := 0 + end := len(e.NeqErrs.errs) + for { + if iter == end { + iter = 0 + + q[0] = nil + q = q[1:] + if len(q) == 0 { + break + } + + end = len(q[0].errs) + continue + } + + v := q[0].errs[iter] + switch v.(type) { + case *noEqualErr: + if v.(*noEqualErr).NeqErrs != nil { + q = append(q, &task{append(q[0].path, v.(*noEqualErr).Path...), v.(*noEqualErr).NeqErrs.errs}) + } else { + msg = append(msg, + fmt.Sprintf("compare: field %s: %s", + strings.Join(append(q[0].path, v.(*noEqualErr).Path...), "."), v.(*noEqualErr).Message)) + } + case *noEqualErrs: + q = append(q, &task{path: q[0].path, errs: v.(*noEqualErrs).errs}) + } + iter++ + } + return strings.Join(msg, "\n") +} + +// newNoEqualErr returns a NoEqualErr. +func newNoEqualErr(field string, err error) error { + noEqerr := &noEqualErr{ + Path: []string{field}, + Message: "", + } + + if err != nil { + if v, ok := err.(*noEqualErr); ok { + noEqerr.Path = append(noEqerr.Path, v.Path...) + if v.Message != "" { + noEqerr.Message = v.Message + } + } else if v, ok := err.(*noEqualErrs); ok { + noEqerr.NeqErrs = v + } else { + noEqerr.Message = err.Error() + } + } + return noEqerr +} + +// JoinErr returns an error that warp the given errors. +func JoinErr(errs ...error) error { + n := 0 + for _, err := range errs { + if err != nil { + n++ + } + } + if n == 0 { + return nil + } + e := &noEqualErrs{ + errs: make([]error, 0, n), + } + for _, err := range errs { + if err != nil { + e.errs = append(e.errs, err) + } + } + return e +} + +func (e *noEqualErrs) Error() string { + var b []byte + for i, err := range e.errs { + if i > 0 { + b = append(b, '\n') + } + b = append(b, err.Error()...) + } + return string(b) +} diff --git a/pkg/compare/error_test.go b/pkg/compare/error_test.go new file mode 100644 index 0000000..c03a4b2 --- /dev/null +++ b/pkg/compare/error_test.go @@ -0,0 +1,35 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package compare + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewNoEqualErr(t *testing.T) { + err := newNoEqualErr("data", fmt.Errorf("this is msg")) + err = newNoEqualErr("to", err) + err = newNoEqualErr("path", err) + assert.Equal(t, "compare: field path.to.data: this is msg", err.Error()) +} diff --git a/pkg/runner/grpc.go b/pkg/runner/grpc.go new file mode 100644 index 0000000..dea7961 --- /dev/null +++ b/pkg/runner/grpc.go @@ -0,0 +1,328 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package runner + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/bufbuild/protocompile" + "github.com/linuxsuren/api-testing/pkg/compare" + "github.com/linuxsuren/api-testing/pkg/testing" + "github.com/tidwall/gjson" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" +) + +type gRPCTestCaseRunner struct { + UnimplementedRunner + host string + proto testing.GRPCDesc +} + +func NewGRPCTestCaseRunner(host string, proto testing.GRPCDesc) TestCaseRunner { + runner := &gRPCTestCaseRunner{ + UnimplementedRunner: NewDefaultUnimplementedRunner(), + host: host, + proto: proto, + } + return runner +} + +func (r *gRPCTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataContext any, ctx context.Context) (output any, err error) { + r.log.Info("start to run: '%s'\n", testcase.Name) + record := NewReportRecord() + defer func(rr *ReportRecord) { + rr.EndTime = time.Now() + rr.Error = err + rr.API = testcase.Request.API + rr.Method = "gRPC" + r.testReporter.PutRecord(rr) + }(record) + + defer func() { + if err == nil { + err = runJob(testcase.After) + } + }() + + contextDir := NewContextKeyBuilder().ParentDir().GetContextValueOrEmpty(ctx) + if err = testcase.Request.Render(dataContext, contextDir); err != nil { + return + } + + md, err := getMethodDescriptor(ctx, r, testcase) + if err != nil { + return nil, err + } + + if err = runJob(testcase.Before); err != nil { + return + } + + r.log.Info("start to send request to %s\n", testcase.Request.API) + conn, err := grpc.Dial(r.host, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + defer conn.Close() + + payload := testcase.Request.Body + respsStr, err := invokeRequest(ctx, md, payload, conn) + if err != nil { + return nil, err + } + + record.Body = strings.Join(respsStr, ",") + r.log.Debug("response body: %s\n", record.Body) + + output, err = verifyResponsePayload(testcase.Name, testcase.Expect, respsStr) + if err != nil { + return nil, err + } + + return output, nil +} + +func invokeRequest(ctx context.Context, md protoreflect.MethodDescriptor, payload string, conn *grpc.ClientConn) (respones []string, err error) { + resps := make([]*dynamicpb.Message, 0) + if md.IsStreamingClient() || md.IsStreamingServer() { + gpayload := gjson.Parse(payload) + if !gpayload.IsArray() { + return nil, fmt.Errorf("payload is not a JSON array") + } + + reqs := make([]*dynamicpb.Message, len(gpayload.Array())) + for i, v := range gpayload.Array() { + req, err := getReqMessagePb(md, v.Raw) + if err != nil { + return nil, err + } + reqs[i] = req + } + + resps, err = invokeRPCStream(ctx, conn, md, reqs) + if err != nil { + return nil, err + } + + } + request, err := getReqMessagePb(md, payload) + if err != nil { + return nil, err + } + + resp, err := invokeRPC(ctx, conn, md, request) + if err != nil { + return nil, err + } + resps = append(resps, resp) + + return buildResponses(resps) +} + +func getReqMessagePb(md protoreflect.MethodDescriptor, message string) (messagepb *dynamicpb.Message, err error) { + request := dynamicpb.NewMessage(md.Input()) + if message != "" { + err := protojson.Unmarshal([]byte(message), request) + if err != nil { + return nil, err + } + } + return request, nil +} + +func buildResponses(resps []*dynamicpb.Message) ([]string, error) { + respsStr := make([]string, 0) + for i := range resps { + respbR, err := protojson.Marshal(resps[i]) + if err != nil { + return nil, err + } + respsStr = append(respsStr, string(respbR)) + } + return respsStr, nil +} + +func getMethodDescriptor(ctx context.Context, r *gRPCTestCaseRunner, testcase *testing.TestCase) (protoreflect.MethodDescriptor, error) { + compiler := protocompile.Compiler{ + Resolver: protocompile.WithStandardImports( + &protocompile.SourceResolver{ + ImportPaths: r.proto.ImportPath, + }, + ), + } + + linker, err := compiler.Compile(ctx, r.proto.ProtoFile) + if err != nil { + return nil, err + } + + fd, err := linker.AsResolver().FindFileByPath(r.proto.ProtoFile) + if err != nil { + return nil, err + } + + api := splitServiceAndMethod(testcase.Request.API) + if len(api) != 2 { + return nil, fmt.Errorf("%s is not a valid gRPC api name", testcase.Request.API) + } + + sd := fd.Services().ByName(protoreflect.Name(api[0])) + if sd == nil { + return nil, fmt.Errorf("grpc service %s is not found in proto %s", api[0], fd.Name()) + } + + md := sd.Methods().ByName(protoreflect.Name(api[1])) + if md == nil { + return nil, fmt.Errorf("method %s is not found in service %s", api[1], api[0]) + } + return md, nil +} + +func splitServiceAndMethod(api string) []string { + return strings.Split(api, ".") +} + +func getMethodName(md protoreflect.MethodDescriptor) string { + return fmt.Sprintf("/%s/%s", md.Parent().FullName(), md.Name()) +} + +// invokeRPC sends an unary RPC to gRPC server. +func invokeRPC(ctx context.Context, conn grpc.ClientConnInterface, method protoreflect.MethodDescriptor, request *dynamicpb.Message) (resp *dynamicpb.Message, err error) { + resp = dynamicpb.NewMessage(method.Output()) + if err := conn.Invoke(ctx, getMethodName(method), request, resp); err != nil { + return nil, err + } + return resp, nil +} + +// invokeRPCStream combine all three types of streaming rpc into a single function. +func invokeRPCStream(ctx context.Context, conn grpc.ClientConnInterface, method protoreflect.MethodDescriptor, requests []*dynamicpb.Message) (resps []*dynamicpb.Message, err error) { + sd := &grpc.StreamDesc{ + StreamName: string(method.Name()), + ServerStreams: method.IsStreamingServer(), + ClientStreams: method.IsStreamingClient(), + } + + s, err := conn.NewStream(ctx, sd, getMethodName(method)) + if err != nil { + return nil, err + } + + i := 0 + +sendLoop: + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + if len(requests) == i { + break sendLoop + } + if err := s.SendMsg(requests[i]); err != nil { + return nil, err + } + i++ + } + } + + if err = s.CloseSend(); err != nil { + return nil, err + } + + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + resp := dynamicpb.NewMessage(method.Output()) + if err = s.RecvMsg(resp); err != nil { + if err == io.EOF { + return resps, nil + } + return nil, err + } + resps = append(resps, resp) + } + } +} + +func verifyResponsePayload(caseName string, expect testing.Response, jsonPayload []string) (output any, err error) { + mapOutput := map[string]any{ + "data": jsonPayload, + } + + if err = payloadFieldsVerify(caseName, expect, jsonPayload); err != nil { + return + } + + err = Verify(expect, mapOutput) + if err != nil { + return nil, err + } + return +} + +func payloadFieldsVerify(caseName string, expect testing.Response, jsonPayload []string) error { + if expect.Body == "" { + return nil + } + + if !gjson.Valid(expect.Body) { + fmt.Printf("expect.Body: %v\n", expect.Body) + return fmt.Errorf("case %s: expect body is not a valid JSON", caseName) + } + + exp := gjson.Parse(expect.Body) + gjsonPayload := make([]gjson.Result, len(jsonPayload)) + for i := range jsonPayload { + gjsonPayload[i] = gjson.Parse(jsonPayload[i]) + } + + if exp.IsArray() { + return compare.Array(caseName, exp.Array(), gjsonPayload) + } + + if exp.IsObject() { + var msg string + for i := range jsonPayload { + err := compare.Object(fmt.Sprintf("%v[%v]", caseName, i), + exp.Map(), gjsonPayload[i].Map()) + if err != nil { + msg += err.Error() + } + } + + if msg != "" { + return fmt.Errorf(msg) + } + return nil + } + return fmt.Errorf("case %s: unknown expect content", caseName) +} diff --git a/pkg/runner/grpc_test.go b/pkg/runner/grpc_test.go new file mode 100644 index 0000000..5d0de49 --- /dev/null +++ b/pkg/runner/grpc_test.go @@ -0,0 +1,23 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package runner + +// TODO diff --git a/pkg/runner/http.go b/pkg/runner/http.go index 0009b59..dc064de 100644 --- a/pkg/runner/http.go +++ b/pkg/runner/http.go @@ -14,9 +14,7 @@ import ( "github.com/andreyvit/diff" "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/linuxsuren/api-testing/pkg/runner/kubernetes" "github.com/linuxsuren/api-testing/pkg/testing" - fakeruntime "github.com/linuxsuren/go-fake-runtime" "github.com/tidwall/gjson" "github.com/xeipuuv/gojsonschema" ) @@ -103,20 +101,17 @@ func (r ReportResultSlice) Swap(i, j int) { } type simpleTestCaseRunner struct { - testReporter TestReporter - writer io.Writer - log LevelWriter - execer fakeruntime.Execer + UnimplementedRunner simpleResponse SimpleResponse } // NewSimpleTestCaseRunner creates the instance of the simple test case runner func NewSimpleTestCaseRunner() TestCaseRunner { - runner := &simpleTestCaseRunner{} - return runner.WithOutputWriter(io.Discard). - WithWriteLevel("info"). - WithTestReporter(NewDiscardTestReporter()). - WithExecer(fakeruntime.DefaultExecer{}) + runner := &simpleTestCaseRunner{ + UnimplementedRunner: NewDefaultUnimplementedRunner(), + simpleResponse: SimpleResponse{}, + } + return runner } // ContextKey is the alias type of string for context key @@ -231,32 +226,6 @@ func (r *simpleTestCaseRunner) RunTestCase(testcase *testing.TestCase, dataConte return } -// WithOutputWriter sets the io.Writer -func (r *simpleTestCaseRunner) WithOutputWriter(writer io.Writer) TestCaseRunner { - r.writer = writer - return r -} - -// WithWriteLevel sets the level writer -func (r *simpleTestCaseRunner) WithWriteLevel(level string) TestCaseRunner { - if level != "" { - r.log = NewDefaultLevelWriter(level, r.writer) - } - return r -} - -// WithTestReporter sets the TestReporter -func (r *simpleTestCaseRunner) WithTestReporter(reporter TestReporter) TestCaseRunner { - r.testReporter = reporter - return r -} - -// WithExecer sets the execer -func (r *simpleTestCaseRunner) WithExecer(execer fakeruntime.Execer) TestCaseRunner { - r.execer = execer - return r -} - func (r *simpleTestCaseRunner) withResponseRecord(resp *http.Response) (responseBodyData []byte, err error) { responseBodyData, err = io.ReadAll(resp.Body) r.simpleResponse = SimpleResponse{ @@ -343,25 +312,7 @@ func verifyResponseBodyData(caseName string, expect testing.Response, responseBo return } - for _, verify := range expect.Verify { - var program *vm.Program - if program, err = expr.Compile(verify, expr.Env(mapOutput), - expr.AsBool(), kubernetes.PodValidatorFunc(), - kubernetes.KubernetesValidatorFunc()); err != nil { - return - } - - var result interface{} - if result, err = expr.Run(program, mapOutput); err != nil { - return - } - - if !result.(bool) { - err = fmt.Errorf("failed to verify: %s", verify) - fmt.Println(err) - break - } - } + err = Verify(expect, mapOutput) return } diff --git a/pkg/runner/http_test.go b/pkg/runner/http_test.go index 82c4f6b..d258731 100644 --- a/pkg/runner/http_test.go +++ b/pkg/runner/http_test.go @@ -341,7 +341,8 @@ func TestTestCase(t *testing.T) { if tt.verify == nil { tt.verify = hasError } - runner := NewSimpleTestCaseRunner().WithOutputWriter(os.Stdout) + runner := NewSimpleTestCaseRunner() + runner.WithOutputWriter(os.Stdout) if tt.execer != nil { runner.WithExecer(tt.execer) } diff --git a/pkg/runner/reporter.go b/pkg/runner/reporter.go index a9069f1..888959e 100644 --- a/pkg/runner/reporter.go +++ b/pkg/runner/reporter.go @@ -9,7 +9,7 @@ type TestReporter interface { ExportAllReportResults() (ReportResultSlice, error) } -// ReportRecord represents the raw data of a HTTP request +// ReportRecord represents the raw data of a request type ReportRecord struct { Method string API string diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index f119beb..6eebafe 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -2,6 +2,7 @@ package runner import ( "context" + "fmt" "io" "github.com/linuxsuren/api-testing/pkg/testing" @@ -11,10 +12,10 @@ import ( // TestCaseRunner represents a test case runner type TestCaseRunner interface { RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) - WithOutputWriter(io.Writer) TestCaseRunner - WithWriteLevel(level string) TestCaseRunner - WithTestReporter(TestReporter) TestCaseRunner - WithExecer(fakeruntime.Execer) TestCaseRunner + WithOutputWriter(io.Writer) + WithWriteLevel(level string) + WithTestReporter(TestReporter) + WithExecer(fakeruntime.Execer) } // HTTPResponseRecord represents a http response record @@ -28,3 +29,51 @@ type SimpleResponse struct { Body string StatusCode int } + +// NewDefaultUnimplementedRunner initializes an unimplementedRunner using the default values. +func NewDefaultUnimplementedRunner() UnimplementedRunner { + return UnimplementedRunner{ + testReporter: NewDiscardTestReporter(), + writer: io.Discard, + log: NewDefaultLevelWriter("info", io.Discard), + execer: fakeruntime.DefaultExecer{}, + } +} + +// UnimplementedRunner implements interface TestCaseRunner except method RunTestCase. +// +// Generally, this struct can be inherited directly when implementing a new runner. +// It is recommended to use NewDefaultUnimplementedRunner to initalize rather than +// to fill it manully. +type UnimplementedRunner struct { + testReporter TestReporter + writer io.Writer + log LevelWriter + execer fakeruntime.Execer +} + +func (r *UnimplementedRunner) RunTestCase(testcase *testing.TestCase, dataContext interface{}, ctx context.Context) (output interface{}, err error) { + return nil, fmt.Errorf("unimplemented") +} + +// WithOutputWriter sets the io.Writer +func (r *UnimplementedRunner) WithOutputWriter(writer io.Writer) { + r.writer = writer +} + +// WithWriteLevel sets the level writer +func (r *UnimplementedRunner) WithWriteLevel(level string) { + if level != "" { + r.log = NewDefaultLevelWriter(level, r.writer) + } +} + +// WithTestReporter sets the TestReporter +func (r *UnimplementedRunner) WithTestReporter(reporter TestReporter) { + r.testReporter = reporter +} + +// WithExecer sets the execer +func (r *UnimplementedRunner) WithExecer(execer fakeruntime.Execer) { + r.execer = execer +} diff --git a/pkg/runner/runner_factory.go b/pkg/runner/runner_factory.go new file mode 100644 index 0000000..b691cc2 --- /dev/null +++ b/pkg/runner/runner_factory.go @@ -0,0 +1,32 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package runner + +import "github.com/linuxsuren/api-testing/pkg/testing" + +// GetTestSuiteRunner returns a proper runner according to the given test suite. +func GetTestSuiteRunner(suite *testing.TestSuite) TestCaseRunner { + // TODO: should be refactored to meet more types of runners + if suite.Spec.GRPC != nil { + return NewGRPCTestCaseRunner(suite.API, *suite.Spec.GRPC) + } + return NewSimpleTestCaseRunner() +} diff --git a/pkg/runner/verify.go b/pkg/runner/verify.go new file mode 100644 index 0000000..a13c9dd --- /dev/null +++ b/pkg/runner/verify.go @@ -0,0 +1,54 @@ +/* +MIT License +Copyright (c) 2023 API Testing Authors. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package runner + +import ( + "fmt" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/linuxsuren/api-testing/pkg/runner/kubernetes" + "github.com/linuxsuren/api-testing/pkg/testing" +) + +// Verify if the data satisfies the expression. +func Verify(expect testing.Response, data map[string]any) (err error) { + for _, verify := range expect.Verify { + var program *vm.Program + if program, err = expr.Compile(verify, expr.Env(data), + expr.AsBool(), kubernetes.PodValidatorFunc(), + kubernetes.KubernetesValidatorFunc()); err != nil { + return err + } + + var result interface{} + if result, err = expr.Run(program, data); err != nil { + return err + } + + if !result.(bool) { + err = fmt.Errorf("failed to verify: %s", verify) + fmt.Println(err) + break + } + } + return +} diff --git a/pkg/server/remote_server.go b/pkg/server/remote_server.go index 577653c..0e90a4e 100644 --- a/pkg/server/remote_server.go +++ b/pkg/server/remote_server.go @@ -199,17 +199,17 @@ func (s *server) Run(ctx context.Context, task *TestTask) (reply *TestResult, er reply = &TestResult{} for _, testCase := range suite.Items { - simpleRunner := runner.NewSimpleTestCaseRunner() - simpleRunner.WithOutputWriter(buf) - simpleRunner.WithWriteLevel(task.Level) + suiteRunner := runner.GetTestSuiteRunner(suite) + suiteRunner.WithOutputWriter(buf) + suiteRunner.WithWriteLevel(task.Level) // reuse the API prefix if strings.HasPrefix(testCase.Request.API, "/") { testCase.Request.API = fmt.Sprintf("%s%s", suite.API, testCase.Request.API) } - output, testErr := simpleRunner.RunTestCase(&testCase, dataContext, ctx) - if getter, ok := simpleRunner.(runner.HTTPResponseRecord); ok { + output, testErr := suiteRunner.RunTestCase(&testCase, dataContext, ctx) + if getter, ok := suiteRunner.(runner.HTTPResponseRecord); ok { resp := getter.GetResponseRecord() reply.TestCaseResult = append(reply.TestCaseResult, &TestCaseResult{ StatusCode: int32(resp.StatusCode), @@ -628,7 +628,7 @@ func (s *server) FunctionsQueryStream(srv Runner_FunctionsQueryStreamServer) err } } if err := srv.Send(reply); err != nil { - return nil + return err } } } diff --git a/pkg/testing/case.go b/pkg/testing/case.go index 8d1b25a..ad2b772 100644 --- a/pkg/testing/case.go +++ b/pkg/testing/case.go @@ -10,8 +10,15 @@ type TestSuite struct { } type APISpec struct { - Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` - URL string `yaml:"url,omitempty" json:"url,omitempty"` + Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` + URL string `yaml:"url,omitempty" json:"url,omitempty"` + GRPC *GRPCDesc `yaml:"grpc,omitempty" json:"grpc,omitempty"` +} +type GRPCDesc struct { + ImportPath []string `yaml:"import,omitempty" json:"import,omitempty"` + ServerReflection bool `yaml:"serverReflection,omitempty" json:"serverReflection,omitempty"` + ProtoFile string `yaml:"protofile,omitempty" json:"protofile,omitempty"` + ProtoSet string `yaml:"protoset,omitempty" json:"protoset,omitempty"` } // TestCase represents a test case diff --git a/sample/grpc-sample.yaml b/sample/grpc-sample.yaml new file mode 100644 index 0000000..5709948 --- /dev/null +++ b/sample/grpc-sample.yaml @@ -0,0 +1,47 @@ +#!api-testing +# yaml-language-server: $schema=https://linuxsuren.github.io/api-testing/api-testing-schema.json +# see also https://github.com/LinuxSuRen/api-testing +name: grpc-sample +api: 127.0.0.1:7070 +spec: + grpc: + import: + - ./pkg/server + protofile: server.proto + serverReflection: false +items: + - name: GetVersion + request: + api: Runner.GetVersion + - name: FunctionsQuery + request: + api: Runner.FunctionsQuery + body: | + { + "name": "hello" + } + expect: + body: | + { + "data": [ + { + "key": "hello", + "value": "func() string" + } + ] + } + - name: FunctionsQueryStream + request: + api: Runner.FunctionsQueryStream + body: | + [ + { + "name": "hello" + }, + { + "name": "title" + } + ] + expect: + verify: + - "len(data) == 2" diff --git a/sample/halo.yaml b/sample/halo.yaml index 682ecb4..e6d0bd6 100644 --- a/sample/halo.yaml +++ b/sample/halo.yaml @@ -4,18 +4,18 @@ name: Halo api: https://demo.halo.run items: -- name: publickey - request: - api: /login/public-key - method: GET -- name: login - request: - api: /login - method: POST - body: | - { - "username": "demo", - "password": "P@ssw0rd123" - } - expect: - statusCode: 500 + - name: publickey + request: + api: /login/public-key + method: GET + - name: login + request: + api: /login + method: POST + body: | + { + "username": "demo", + "password": "P@ssw0rd123" + } + expect: + statusCode: 500 diff --git a/sample/kubernetes.yaml b/sample/kubernetes.yaml index 23416e7..6ad695c 100644 --- a/sample/kubernetes.yaml +++ b/sample/kubernetes.yaml @@ -4,86 +4,86 @@ name: Kubernetes api: | {{default "https://172.11.0.18:6443" (env "SERVER")}} items: -- name: pods - request: - api: /api/v1/namespaces/kube-system/pods - header: - Authorization: Bearer {{env "K8S_TOKEN"}} - expect: - verify: - - data.kind == "PodList" - - pod("kube-system", "kube-ovn-cni-55bz9").Exist() - - k8s("pods", "kube-system", "kube-ovn-cni-55bz9").Exist() - - k8s("deployments", "kube-system", "coredns").Exist() - - k8s("deployments", "kube-system", "coredns").ExpectField(2, "spec", "replicas") - - k8s("deployments", "kube-system", "coredns").ExpectField("kube-dns", "metadata", "labels", "k8s-app") - - k8s("daemonsets", "kube-system", "kube-ovn-cni").Exist() - - k8s({"kind":"virtualmachines","group":"kubevirt.io"}, "vm-test", "vm-win10-dkkhl").Exist() -- name: create-configmap - request: - api: /api/v1/namespaces/default/configmaps - header: - Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkRINXBRRi0zSURrbkRDWGhfVHpEaGFuOVdpcEVLSmFwYUI4Y1V5YjFpcUEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJjbHVzdGVyLWFkbWluLXRva2VuLWtobnI0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImNsdXN0ZXItYWRtaW4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJmZmNlODg0Ny0yZGY4LTQyMTktOGRjYS1mNGRlMWYzNWNmYzkiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06Y2x1c3Rlci1hZG1pbiJ9.YapUNL7aSlAzlZwDqcMF1-eNpaEs0ZPwybV1uM289fDk8RwjHpLQzVZV0IewaOCAjifwyTyqs1Vgd4nF9I7CYPv64cjMcVTQHCj_-pAxXjiYEM9LkR_b__WGsd-3Z0aRrdyO4WS7moRxZ4kz7ULd_OtlHpq-cFIQtytOaQSZNSbxpa5uP7g7y-uv0nwXBSwqZL9j5XimGlYyy999Q8Vc2GLDrDdVp69wuvToODQzJV44nfuA_dhUFQOzC4sE7Dkq7JarrvZspstqLo1ULzt_Z-cZ-qAu_pUaLHkoLZH5o97g4UF8AXeFYLj8YP_IBP9uhDrm829pNHU82N6Hn-80NQ - method: POST - body: | - { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "config", - "namespace": "default", - "labels": { - "key": "{{randomKubernetesName}}" + - name: pods + request: + api: /api/v1/namespaces/kube-system/pods + header: + Authorization: Bearer {{env "K8S_TOKEN"}} + expect: + verify: + - data.kind == "PodList" + - pod("kube-system", "kube-ovn-cni-55bz9").Exist() + - k8s("pods", "kube-system", "kube-ovn-cni-55bz9").Exist() + - k8s("deployments", "kube-system", "coredns").Exist() + - k8s("deployments", "kube-system", "coredns").ExpectField(2, "spec", "replicas") + - k8s("deployments", "kube-system", "coredns").ExpectField("kube-dns", "metadata", "labels", "k8s-app") + - k8s("daemonsets", "kube-system", "kube-ovn-cni").Exist() + - k8s({"kind":"virtualmachines","group":"kubevirt.io"}, "vm-test", "vm-win10-dkkhl").Exist() + - name: create-configmap + request: + api: /api/v1/namespaces/default/configmaps + header: + Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkRINXBRRi0zSURrbkRDWGhfVHpEaGFuOVdpcEVLSmFwYUI4Y1V5YjFpcUEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJjbHVzdGVyLWFkbWluLXRva2VuLWtobnI0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImNsdXN0ZXItYWRtaW4iLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJmZmNlODg0Ny0yZGY4LTQyMTktOGRjYS1mNGRlMWYzNWNmYzkiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06Y2x1c3Rlci1hZG1pbiJ9.YapUNL7aSlAzlZwDqcMF1-eNpaEs0ZPwybV1uM289fDk8RwjHpLQzVZV0IewaOCAjifwyTyqs1Vgd4nF9I7CYPv64cjMcVTQHCj_-pAxXjiYEM9LkR_b__WGsd-3Z0aRrdyO4WS7moRxZ4kz7ULd_OtlHpq-cFIQtytOaQSZNSbxpa5uP7g7y-uv0nwXBSwqZL9j5XimGlYyy999Q8Vc2GLDrDdVp69wuvToODQzJV44nfuA_dhUFQOzC4sE7Dkq7JarrvZspstqLo1ULzt_Z-cZ-qAu_pUaLHkoLZH5o97g4UF8AXeFYLj8YP_IBP9uhDrm829pNHU82N6Hn-80NQ + method: POST + body: | + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "config", + "namespace": "default", + "labels": { + "key": "{{randomKubernetesName}}" + } + }, + "data": { + "key": "value" } - }, - "data": { - "key": "value" } - } - expect: - statusCode: 201 -- name: update-configmap - request: - api: /api/v1/namespaces/default/configmaps/config - header: - Authorization: Bearer {{env "K8S_TOKEN"}} - method: PUT - body: | - { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "config", - "namespace": "default" - }, - "data": { - "key": "new value" + expect: + statusCode: 201 + - name: update-configmap + request: + api: /api/v1/namespaces/default/configmaps/config + header: + Authorization: Bearer {{env "K8S_TOKEN"}} + method: PUT + body: | + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "config", + "namespace": "default" + }, + "data": { + "key": "new value" + } } - } -- name: get-configmap - request: - api: /api/v1/namespaces/default/configmaps/config - header: - Authorization: Bearer {{env "K8S_TOKEN"}} - method: PUT - body: | - { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": "config", - "namespace": "default" - }, - "data": { - "key": "new value" + - name: get-configmap + request: + api: /api/v1/namespaces/default/configmaps/config + header: + Authorization: Bearer {{env "K8S_TOKEN"}} + method: PUT + body: | + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "config", + "namespace": "default" + }, + "data": { + "key": "new value" + } } - } - expect: - bodyFieldsExpect: - "data/key": "new value" -- name: delete-configmap - request: - api: /api/v1/namespaces/default/configmaps/config - header: - Authorization: Bearer {{env "K8S_TOKEN"}} - method: DELETE + expect: + bodyFieldsExpect: + "data/key": "new value" + - name: delete-configmap + request: + api: /api/v1/namespaces/default/configmaps/config + header: + Authorization: Bearer {{env "K8S_TOKEN"}} + method: DELETE