From 8d459a9eed5b87fdb2c2ac54e50c598a296d15e4 Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Sat, 29 Jul 2023 22:14:58 +0800 Subject: [PATCH] feat: support add multiple stores (#147) --- .gitpod.yml | 3 + Makefile | 10 +- cmd/root.go | 2 +- cmd/root_test.go | 8 +- cmd/server.go | 56 +- cmd/service.go | 35 +- cmd/service/constant.go | 9 + cmd/service_test.go | 6 + .../cypress/e2e/component-exist.cy.ts | 6 + console/atest-ui/src/App.vue | 121 ++- console/atest-ui/src/views/TestCase.vue | 20 +- console/atest-ui/src/views/TestSuite.vue | 16 +- docs/README.md | 31 +- extensions/store-orm/main.go | 150 ++- extensions/store-orm/pkg/convert.go | 36 +- extensions/store-orm/pkg/convert_test.go | 39 +- extensions/store-orm/pkg/types.go | 1 + go.mod | 2 + go.sum | 61 ++ pkg/server/constant.go | 5 + pkg/server/convert.go | 43 + pkg/server/convert_test.go | 62 ++ pkg/server/gateway.go | 15 + pkg/server/gateway_test.go | 20 + pkg/server/remote_server.go | 99 +- pkg/server/remote_server_test.go | 10 +- pkg/server/server.pb.go | 773 +++++++++++---- pkg/server/server.pb.gw.go | 935 +++++++++++++----- pkg/server/server.proto | 64 +- pkg/server/server_grpc.pb.go | 398 ++++++-- pkg/testing/loader_file.go | 9 +- pkg/testing/remote/context.go | 53 + pkg/testing/remote/context_test.go | 52 + pkg/testing/remote/converter.go | 29 +- pkg/testing/remote/converter_test.go | 11 +- pkg/testing/remote/grpc.go | 46 +- pkg/testing/remote/grpc_test.go | 15 +- pkg/testing/remote/loader.pb.go | 779 ++------------- pkg/testing/remote/loader.proto | 66 +- pkg/testing/remote/loader_grpc.pb.go | 99 +- pkg/testing/store.go | 145 +++ pkg/testing/store_test.go | 128 +++ pkg/testing/testdata/stores.yaml | 8 + 43 files changed, 2926 insertions(+), 1550 deletions(-) create mode 100644 .gitpod.yml create mode 100644 cmd/service/constant.go create mode 100644 pkg/server/constant.go create mode 100644 pkg/server/convert.go create mode 100644 pkg/server/convert_test.go create mode 100644 pkg/server/gateway.go create mode 100644 pkg/server/gateway_test.go create mode 100644 pkg/testing/remote/context.go create mode 100644 pkg/testing/remote/context_test.go create mode 100644 pkg/testing/store.go create mode 100644 pkg/testing/store_test.go create mode 100644 pkg/testing/testdata/stores.yaml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..e032143 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,3 @@ +tasks: + - init: make build + command: docker run --pull always --network host ghcr.io/linuxsuren/api-testing:master diff --git a/Makefile b/Makefile index 37bdc99..1cd5159 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,11 @@ install-precheck: cp .github/pre-commit .git/hooks/pre-commit grpc: - protoc --go_out=. --go_opt=paths=source_relative \ + protoc --proto_path=. \ + --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - pkg/server/server.proto - - protoc --go_out=. --go_opt=paths=source_relative \ - --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - pkg/testing/remote/loader.proto + pkg/server/server.proto \ + pkg/testing/remote/loader.proto grpc-gw: protoc -I . --grpc-gateway_out . \ --grpc-gateway_opt logtostderr=true \ diff --git a/cmd/root.go b/cmd/root.go index 193bb5a..41c87fe 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,7 +20,7 @@ func NewRootCmd(execer fakeruntime.Execer, gRPCServer gRPCServer, c.Version = version.GetVersion() c.AddCommand(createInitCommand(execer), createRunCommand(), createSampleCmd(), - createServerCmd(gRPCServer, httpServer), createJSONSchemaCmd(), + createServerCmd(execer, gRPCServer, httpServer), createJSONSchemaCmd(), createServiceCommand(execer), createFunctionCmd()) return } diff --git a/cmd/root_test.go b/cmd/root_test.go index fa5001b..0cd8427 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -10,17 +10,19 @@ import ( ) func TestCreateRunCommand(t *testing.T) { + execer := fakeruntime.FakeExecer{} + cmd := createRunCommand() assert.Equal(t, "run", cmd.Use) - init := createInitCommand(fakeruntime.FakeExecer{}) + init := createInitCommand(execer) assert.Equal(t, "init", init.Use) - s := createServerCmd(&fakeGRPCServer{}, server.NewFakeHTTPServer()) + s := createServerCmd(execer, &fakeGRPCServer{}, server.NewFakeHTTPServer()) assert.NotNil(t, s) assert.Equal(t, "server", s.Use) - root := NewRootCmd(fakeruntime.FakeExecer{}, NewFakeGRPCServer(), server.NewFakeHTTPServer()) + root := NewRootCmd(execer, NewFakeGRPCServer(), server.NewFakeHTTPServer()) root.SetArgs([]string{"init", "-k=demo.yaml", "--wait-namespace", "demo", "--wait-resource", "demo"}) err := root.Execute() assert.Nil(t, err) diff --git a/cmd/server.go b/cmd/server.go index 8ae26ac..4370d65 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -3,11 +3,11 @@ package cmd import ( "bytes" - "errors" "fmt" "log" "net" "net/http" + "os" "path" "strings" "time" @@ -19,43 +19,51 @@ import ( "github.com/linuxsuren/api-testing/pkg/testing" "github.com/linuxsuren/api-testing/pkg/testing/remote" "github.com/linuxsuren/api-testing/pkg/util" + fakeruntime "github.com/linuxsuren/go-fake-runtime" "github.com/spf13/cobra" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) -func createServerCmd(gRPCServer gRPCServer, httpServer server.HTTPServer) (c *cobra.Command) { +func createServerCmd(execer fakeruntime.Execer, gRPCServer gRPCServer, httpServer server.HTTPServer) (c *cobra.Command) { opt := &serverOption{ gRPCServer: gRPCServer, httpServer: httpServer, + execer: execer, } c = &cobra.Command{ - Use: "server", - Short: "Run as a server mode", - RunE: opt.runE, + Use: "server", + Short: "Run as a server mode", + PreRunE: opt.preRunE, + RunE: opt.runE, } flags := c.Flags() flags.IntVarP(&opt.port, "port", "p", 7070, "The RPC server port") flags.IntVarP(&opt.httpPort, "http-port", "", 8080, "The HTTP server port") flags.BoolVarP(&opt.printProto, "print-proto", "", false, "Print the proto content and exit") - flags.StringVarP(&opt.storage, "storage", "", "local", "The storage type, local or etcd") flags.StringArrayVarP(&opt.localStorage, "local-storage", "", []string{"*.yaml"}, "The local storage path") - flags.StringVarP(&opt.grpcStorage, "grpc-storage", "", "", "The grpc storage address") flags.StringVarP(&opt.consolePath, "console-path", "", "", "The path of the console") + flags.StringVarP(&opt.configDir, "config-dir", "", "$HOME/.config/atest", "The config directory") return } type serverOption struct { gRPCServer gRPCServer httpServer server.HTTPServer + execer fakeruntime.Execer port int httpPort int printProto bool - storage string localStorage []string - grpcStorage string consolePath string + configDir string +} + +func (o *serverOption) preRunE(cmd *cobra.Command, args []string) (err error) { + o.configDir = os.ExpandEnv(o.configDir) + err = o.execer.MkdirAll(o.configDir, 0755) + return } func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { @@ -79,30 +87,15 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { return } - var loader testing.Writer - switch o.storage { - case "local": - loader = testing.NewFileWriter("") - for _, storage := range o.localStorage { - if err = loader.Put(storage); err != nil { - break - } + loader := testing.NewFileWriter("") + for _, storage := range o.localStorage { + if loadErr := loader.Put(storage); loadErr != nil { + cmd.PrintErrf("failed to load %s, error: %v\n", storage, loadErr) + continue } - case "grpc": - if o.grpcStorage == "" { - err = errors.New("grpc storage address is required") - return - } - loader, err = remote.NewGRPCLoader(o.grpcStorage) - default: - err = errors.New("invalid storage type") } - if err != nil { - return - } - - removeServer := server.NewRemoteServer(loader) + removeServer := server.NewRemoteServer(loader, remote.NewGRPCloaderFromStore(), o.configDir) s := o.gRPCServer go func() { if gRPCServer, ok := s.(reflection.GRPCServer); ok { @@ -113,13 +106,14 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { s.Serve(lis) }() - mux := runtime.NewServeMux() + mux := runtime.NewServeMux(runtime.WithMetadata(server.MetadataStoreFunc)) // runtime.WithIncomingHeaderMatcher(func(key string) (s string, b bool) { err = server.RegisterRunnerHandlerServer(cmd.Context(), mux, removeServer) if err == nil { mux.HandlePath(http.MethodGet, "/", frontEndHandlerWithLocation(o.consolePath)) mux.HandlePath(http.MethodGet, "/assets/{asset}", frontEndHandlerWithLocation(o.consolePath)) mux.HandlePath(http.MethodGet, "/healthz", frontEndHandlerWithLocation(o.consolePath)) o.httpServer.WithHandler(mux) + log.Printf("HTTP server listening at %v", httplis.Addr()) err = o.httpServer.Serve(httplis) } return diff --git a/cmd/service.go b/cmd/service.go index 202608a..f22c5b3 100644 --- a/cmd/service.go +++ b/cmd/service.go @@ -9,6 +9,7 @@ import ( _ "embed" + "github.com/linuxsuren/api-testing/cmd/service" "github.com/linuxsuren/api-testing/pkg/version" fakeruntime "github.com/linuxsuren/go-fake-runtime" "github.com/spf13/cobra" @@ -272,26 +273,23 @@ type linuxService struct { commonService } -const systemCtl = "systemctl" -const serviceName = "atest" - func (s *linuxService) Start() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "start", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "start", service.ServiceName) return } func (s *linuxService) Stop() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "stop", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "stop", service.ServiceName) return } func (s *linuxService) Restart() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "restart", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "restart", service.ServiceName) return } func (s *linuxService) Status() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "status", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "status", service.ServiceName) if err != nil && err.Error() == "exit status 3" { // this is normal case err = nil @@ -301,13 +299,13 @@ func (s *linuxService) Status() (output string, err error) { func (s *linuxService) Install() (output string, err error) { if err = os.WriteFile(s.scriptPath, []byte(s.script), os.ModeAppend); err == nil { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "enable", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "enable", service.ServiceName) } return } func (s *linuxService) Uninstall() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn(systemCtl, "", "disable", serviceName) + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "disable", service.ServiceName) return } @@ -324,7 +322,7 @@ type containerService struct { const defaultImage = "ghcr.io/linuxsuren/api-testing" -func newContainerService(execer fakeruntime.Execer, client, image, tag, localStorage string, writer io.Writer) (service Service) { +func newContainerService(execer fakeruntime.Execer, client, image, tag, localStorage string, writer io.Writer) (svc Service) { if tag == "" { tag = "latest" } @@ -335,7 +333,7 @@ func newContainerService(execer fakeruntime.Execer, client, image, tag, localSto containerServer := &containerService{ Execer: execer, client: client, - name: serviceName, + name: service.ServiceName, image: image, tag: tag, localStorage: localStorage, @@ -344,11 +342,11 @@ func newContainerService(execer fakeruntime.Execer, client, image, tag, localSto } if strings.HasSuffix(client, ServiceModePodman.String()) { - service = &podmanService{ + svc = &podmanService{ containerService: containerServer, } } else { - service = containerServer + svc = containerServer } return } @@ -426,14 +424,19 @@ func (s *podmanService) Start() (output string, err error) { return } +func (s *podmanService) Stop() (output string, err error) { + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "stop", service.PodmanServiceName) + return +} + func (s *podmanService) installService() (output string, err error) { output, err = s.Execer.RunCommandAndReturn(s.client, "", "generate", "systemd", "--new", "--files", "--name", s.name) if err == nil { var result string - result, err = s.Execer.RunCommandAndReturn("mv", "", "container-atest.service", "/etc/systemd/system") + result, err = s.Execer.RunCommandAndReturn("mv", "", service.PodmanServiceName, "/etc/systemd/system") if err == nil { output = fmt.Sprintf("%s\n%s", output, result) - if result, err = s.Execer.RunCommandAndReturn("systemctl", "", "enable", "container-atest.service"); err == nil { + if result, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "enable", service.PodmanServiceName); err == nil { output = fmt.Sprintf("%s\n%s", output, result) } } @@ -453,6 +456,6 @@ func (s *podmanService) Uninstall() (output string, err error) { } func (s *podmanService) uninstallService() (output string, err error) { - output, err = s.Execer.RunCommandAndReturn("systemctl", "", "disable", "container-atest.service") + output, err = s.Execer.RunCommandAndReturn(service.SystemCtl, "", "disable", service.PodmanServiceName) return } diff --git a/cmd/service/constant.go b/cmd/service/constant.go new file mode 100644 index 0000000..f084f2b --- /dev/null +++ b/cmd/service/constant.go @@ -0,0 +1,9 @@ +package service + +const ( + // PodmanServiceName is the service name of podman service + PodmanServiceName = "container-atest.service" + + SystemCtl = "systemctl" + ServiceName = "atest" +) diff --git a/cmd/service_test.go b/cmd/service_test.go index f037aee..5c9673d 100644 --- a/cmd/service_test.go +++ b/cmd/service_test.go @@ -155,6 +155,12 @@ func TestService(t *testing.T) { targetOS: fakeruntime.OSLinux, mode: ServiceModePodman.String(), expectOutput: "", + }, { + name: "stop in podman", + action: "stop", + targetOS: fakeruntime.OSLinux, + mode: ServiceModePodman.String(), + expectOutput: "", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/console/atest-ui/cypress/e2e/component-exist.cy.ts b/console/atest-ui/cypress/e2e/component-exist.cy.ts index b3c685d..63596ad 100644 --- a/console/atest-ui/cypress/e2e/component-exist.cy.ts +++ b/console/atest-ui/cypress/e2e/component-exist.cy.ts @@ -4,6 +4,7 @@ console.log(Cypress.browser) describe('Suite Manage', () => { const suiteName = userID_Alpha() + const store = "local" const sampleAPIAddress = "http://foo" console.log(sampleAPIAddress) @@ -13,6 +14,11 @@ describe('Suite Manage', () => { cy.get('[test-id="open-new-suite-dialog"]').click() + const storeSelect = cy.get('[test-id=suite-form-store] input') + storeSelect.click() + storeSelect.type(store) + storeSelect.trigger('keydown', {key: 'Enter'}) + cy.get('[test-id=suite-form-name]').should('be.visible').type(suiteName) cy.get('[test-id=suite-form-api]').should('be.visible').type(sampleAPIAddress) cy.get('[test-id=suite-form-submit]').should('be.visible').click() diff --git a/console/atest-ui/src/App.vue b/console/atest-ui/src/App.vue index 3a28e89..100e590 100644 --- a/console/atest-ui/src/App.vue +++ b/console/atest-ui/src/App.vue @@ -4,25 +4,32 @@ import TestSuite from './views/TestSuite.vue' import TemplateFunctions from './views/TemplateFunctions.vue' import { reactive, ref, watch } from 'vue' import { ElTree } from 'element-plus' -import type { FormInstance } from 'element-plus' +import type { FormInstance, FormRules } from 'element-plus' import { Edit } from '@element-plus/icons-vue' +import type { Suite } from './types' interface Tree { id: string label: string parent: string + store: string children?: Tree[] } const testCaseName = ref('') const testSuite = ref('') +const store = ref('') const handleNodeClick = (data: Tree) => { if (data.children) { viewName.value = 'testsuite' testSuite.value = data.label + store.value = data.store const requestOptions = { method: 'POST', + headers: { + 'X-Store-Name': data.store + }, body: JSON.stringify({ name: data.label }) @@ -36,6 +43,7 @@ const handleNodeClick = (data: Tree) => { data.children?.push({ id: data.label + item.name, label: item.name, + store: data.store, parent: data.label } as Tree) }) @@ -44,6 +52,7 @@ const handleNodeClick = (data: Tree) => { } else { testCaseName.value = data.label testSuite.value = data.parent + store.value = data.store viewName.value = 'testcase' } } @@ -52,14 +61,16 @@ const data = ref([] as Tree[]) const treeRef = ref>() const currentNodekey = ref('') -function loadTestSuites() { +function loadTestSuites(store: string) { const requestOptions = { - method: 'POST' + method: 'POST', + headers: { + 'X-Store-Name': store + }, } fetch('/server.Runner/GetSuites', requestOptions) .then((response) => response.json()) .then((d) => { - data.value = [] as Tree[] if (!d.data) { return } @@ -67,6 +78,7 @@ function loadTestSuites() { let suite = { id: k, label: k, + store: store, children: [] as Tree[] } as Tree @@ -74,6 +86,7 @@ function loadTestSuites() { suite.children?.push({ id: k + item, label: item, + store: store, parent: k } as Tree) }) @@ -95,40 +108,76 @@ function loadTestSuites() { } }) } -loadTestSuites() + +interface Store { + name: string, + description: string, +} + +const stores = ref([] as Store[]) +function loadStores() { + const requestOptions = { + method: 'POST', + } + fetch('/server.Runner/GetStores', requestOptions) + .then((response) => response.json()) + .then((d) => { + stores.value = d.data + data.value = [] as Tree[] + + d.data.forEach((item: any) => { + loadTestSuites(item.name) + }) + }) +} +loadStores() const dialogVisible = ref(false) const suiteCreatingLoading = ref(false) const suiteFormRef = ref() const testSuiteForm = reactive({ name: '', - api: '' + api: '', + store: '' }) function openTestSuiteCreateDialog() { dialogVisible.value = true } -const submitForm = (formEl: FormInstance | undefined) => { +const rules = reactive>({ + name: [{ required: true, message: 'Name is required', trigger: 'blur' }], + store: [{ required: true, message: 'Location is required', trigger: 'blur' }] +}) +const submitForm = async (formEl: FormInstance | undefined) => { if (!formEl) return - suiteCreatingLoading.value = true + console.log(formEl) + await formEl.validate((valid: boolean, fields) => { + console.log(valid, fields) + if (valid) { + suiteCreatingLoading.value = true - const requestOptions = { - method: 'POST', - body: JSON.stringify({ - name: testSuiteForm.name, - api: testSuiteForm.api - }) - } + const requestOptions = { + method: 'POST', + headers: { + 'X-Store-Name': testSuiteForm.store + }, + body: JSON.stringify({ + name: testSuiteForm.name, + api: testSuiteForm.api + }) + } - fetch('/server.Runner/CreateTestSuite', requestOptions) - .then((response) => response.json()) - .then(() => { - suiteCreatingLoading.value = false - loadTestSuites() - }) - - dialogVisible.value = false + fetch('/server.Runner/CreateTestSuite', requestOptions) + .then((response) => response.json()) + .then(() => { + suiteCreatingLoading.value = false + loadStores() + dialogVisible.value = false + formEl.resetFields() + }) + } + }) } const filterText = ref('') @@ -166,14 +215,16 @@ const viewName = ref('testcase') @@ -182,7 +233,25 @@ const viewName = ref('testcase')