From 91ae93a9c9636c61774eb25459da5d724ca71a03 Mon Sep 17 00:00:00 2001 From: rick Date: Wed, 5 Mar 2025 10:32:24 +0000 Subject: [PATCH] support to load swaggers from an extension --- cmd/extension.go | 118 ++++++++++----------- cmd/server.go | 11 ++ console/atest-ui/src/views/TestSuite.vue | 8 +- console/atest-ui/src/views/net.ts | 8 +- console/atest-ui/src/views/types.ts | 22 ++++ pkg/apispec/remote_swagger.go | 127 +++++++++++++++++++++++ pkg/apispec/swagger.go | 17 +++ pkg/downloader/extension.go | 11 +- pkg/downloader/oci.go | 2 + pkg/util/home/common.go | 6 +- pkg/util/home/common_test.go | 29 ++++++ 11 files changed, 293 insertions(+), 66 deletions(-) create mode 100644 pkg/apispec/remote_swagger.go create mode 100644 pkg/util/home/common_test.go diff --git a/cmd/extension.go b/cmd/extension.go index 166dcd3..b61ca02 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -17,75 +17,75 @@ limitations under the License. package cmd import ( - "fmt" - "io" - "path/filepath" - "runtime" - "time" + "fmt" + "io" + "path/filepath" + "runtime" + "time" - "github.com/linuxsuren/api-testing/pkg/downloader" - "github.com/spf13/cobra" + "github.com/linuxsuren/api-testing/pkg/downloader" + "github.com/spf13/cobra" ) type extensionOption struct { - ociDownloader downloader.PlatformAwareOCIDownloader - output string - registry string - kind string - tag string - os string - arch string - timeout time.Duration - imagePrefix string + ociDownloader downloader.PlatformAwareOCIDownloader + output string + registry string + kind string + tag string + os string + arch string + timeout time.Duration + imagePrefix string } func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader) (c *cobra.Command) { - opt := &extensionOption{ - ociDownloader: ociDownloader, - } - c = &cobra.Command{ - Use: "extension", - Short: "Download extension binary files", - Long: "Download the store extension files", - Args: cobra.MinimumNArgs(1), - RunE: opt.runE, - } - flags := c.Flags() - flags.StringVarP(&opt.output, "output", "", ".", "The target directory") - flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty") - flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io") - flags.StringVarP(&opt.kind, "kind", "", "store", "The extension kind") - flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS") - flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture") - flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading") - flags.StringVarP(&opt.imagePrefix, "image-prefix", "", "linuxsuren", "The prefix for the image address") - return + opt := &extensionOption{ + ociDownloader: ociDownloader, + } + c = &cobra.Command{ + Use: "extension", + Short: "Download extension binary files", + Long: "Download the store extension files", + Args: cobra.MinimumNArgs(1), + RunE: opt.runE, + } + flags := c.Flags() + flags.StringVarP(&opt.output, "output", "", ".", "The target directory") + flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty") + flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io") + flags.StringVarP(&opt.kind, "kind", "", "store", "The extension kind") + flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS") + flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture") + flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading") + flags.StringVarP(&opt.imagePrefix, "image-prefix", "", "linuxsuren", "The prefix for the image address") + return } func (o *extensionOption) runE(cmd *cobra.Command, args []string) (err error) { - o.ociDownloader.WithOS(o.os) - o.ociDownloader.WithArch(o.arch) - o.ociDownloader.WithRegistry(o.registry) - o.ociDownloader.WithImagePrefix(o.imagePrefix) - o.ociDownloader.WithTimeout(o.timeout) - o.ociDownloader.WithKind(o.kind) - o.ociDownloader.WithContext(cmd.Context()) + o.ociDownloader.WithOS(o.os) + o.ociDownloader.WithArch(o.arch) + o.ociDownloader.WithRegistry(o.registry) + o.ociDownloader.WithImagePrefix(o.imagePrefix) + o.ociDownloader.WithTimeout(o.timeout) + o.ociDownloader.WithKind(o.kind) + o.ociDownloader.WithContext(cmd.Context()) - for _, arg := range args { - var reader io.Reader - if reader, err = o.ociDownloader.Download(arg, o.tag, ""); err != nil { - return - } else if reader == nil { - err = fmt.Errorf("cannot find %s", arg) - return - } - extFile := o.ociDownloader.GetTargetFile() - cmd.Println("found target file", extFile) + for _, arg := range args { + var reader io.Reader + if reader, err = o.ociDownloader.Download(arg, o.tag, ""); err != nil { + return + } else if reader == nil { + err = fmt.Errorf("cannot find %s", arg) + return + } + extFile := o.ociDownloader.GetTargetFile() + cmd.Println("found target file", extFile) - targetFile := filepath.Base(extFile) - if err = downloader.WriteTo(reader, o.output, targetFile); err == nil { - cmd.Println("downloaded", targetFile) - } - } - return + targetFile := filepath.Base(extFile) + if err = downloader.WriteTo(reader, o.output, targetFile); err == nil { + cmd.Println("downloaded", targetFile) + } + } + return } diff --git a/cmd/server.go b/cmd/server.go index 8a904d1..35f6374 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "github.com/linuxsuren/api-testing/pkg/apispec" "net" "net/http" "os" @@ -302,6 +303,15 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { _ = o.httpServer.Shutdown(ctx) }() + go func() { + err := apispec.DownloadSwaggerData("", extDownloader) + if err != nil { + fmt.Println("failed to download swagger data", err) + } else { + fmt.Println("success to download swagger data") + } + }() + mux := runtime.NewServeMux(runtime.WithMetadata(server.MetadataStoreFunc), runtime.WithMarshalerOption("application/json+pretty", &runtime.JSONPb{ MarshalOptions: protojson.MarshalOptions{ @@ -342,6 +352,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { mux.HandlePath(http.MethodGet, "/get", o.getAtestBinary) mux.HandlePath(http.MethodPost, "/runner/{suite}/{case}", service.WebRunnerHandler) mux.HandlePath(http.MethodGet, "/api/v1/sbom", service.SBomHandler) + mux.HandlePath(http.MethodGet, "/api/v1/swaggers", apispec.SwaggersHandler) postRequestProxyFunc := postRequestProxy(o.skyWalking) mux.HandlePath(http.MethodPost, "/browser/{app}", postRequestProxyFunc) diff --git a/console/atest-ui/src/views/TestSuite.vue b/console/atest-ui/src/views/TestSuite.vue index d4c9806..3489d8c 100644 --- a/console/atest-ui/src/views/TestSuite.vue +++ b/console/atest-ui/src/views/TestSuite.vue @@ -4,7 +4,7 @@ import { reactive, ref, watch } from 'vue' import { Edit, CopyDocument, Delete } from '@element-plus/icons-vue' import type { FormInstance, FormRules } from 'element-plus' import type { Suite, TestCase, Pair } from './types' -import { NewSuggestedAPIsQuery, GetHTTPMethods } from './types' +import { NewSuggestedAPIsQuery, GetHTTPMethods, SwaggerSuggestion } from './types' import EditButton from '../components/EditButton.vue' import { Cache } from './cache' import { useI18n } from 'vue-i18n' @@ -20,6 +20,7 @@ const props = defineProps({ }) const emit = defineEmits(['updated']) let querySuggestedAPIs = NewSuggestedAPIsQuery(Cache.GetCurrentStore().name, props.name!) +const querySwaggers = SwaggerSuggestion() const suite = ref({ name: '', @@ -325,7 +326,10 @@ const renameTestSuite = (name: string) => { - + diff --git a/console/atest-ui/src/views/net.ts b/console/atest-ui/src/views/net.ts index 6553e27..ee632eb 100644 --- a/console/atest-ui/src/views/net.ts +++ b/console/atest-ui/src/views/net.ts @@ -620,6 +620,12 @@ function GetSuggestedAPIs(name: string, .then(callback) } +function GetSwaggers(callback: (d: any) => void) { + fetch(`/api/v1/swaggers`, {}) + .then(DefaultResponseProcess) + .then(callback) +} + function ReloadMockServer(config: any) { const requestOptions = { method: 'POST', @@ -812,7 +818,7 @@ export const API = { CreateOrUpdateStore, GetStores, DeleteStore, VerifyStore, FunctionsQuery, GetSecrets, DeleteSecret, CreateOrUpdateSecret, - GetSuggestedAPIs, + GetSuggestedAPIs, GetSwaggers, ReloadMockServer, GetMockConfig, SBOM, DataQuery, getToken } diff --git a/console/atest-ui/src/views/types.ts b/console/atest-ui/src/views/types.ts index dd97ff7..66afe23 100644 --- a/console/atest-ui/src/views/types.ts +++ b/console/atest-ui/src/views/types.ts @@ -102,6 +102,28 @@ export function NewSuggestedAPIsQuery(store: string, suite: string) { }) } } + +interface SwaggerItem { + value: string +} + +export function SwaggerSuggestion() { + return function (queryString: string, cb: (arg: any) => void) { + API.GetSwaggers((e) => { + var swaggers = [] as SwaggerItem[] + e.forEach((item: string) => { + swaggers.push({ + "value": `atest://${item}` + }) + }) + + const results = queryString ? swaggers.filter((item: SwaggerItem) => { + return item.value.toLowerCase().indexOf(queryString.toLowerCase()) != -1 + }) : swaggers + cb(results.slice(0, 10)) + }) + } +} export function CreateFilter(queryString: string) { return (v: Pair) => { return v.value.toLowerCase().indexOf(queryString.toLowerCase()) !== -1 diff --git a/pkg/apispec/remote_swagger.go b/pkg/apispec/remote_swagger.go new file mode 100644 index 0000000..3a0be27 --- /dev/null +++ b/pkg/apispec/remote_swagger.go @@ -0,0 +1,127 @@ +/* +Copyright 2025 API Testing Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apispec + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "fmt" + "github.com/linuxsuren/api-testing/pkg/downloader" + "github.com/linuxsuren/api-testing/pkg/util/home" + "io" + "net/http" + "os" + "path/filepath" +) + +func DownloadSwaggerData(output string, dw downloader.PlatformAwareOCIDownloader) (err error) { + dw.WithKind("data") + dw.WithOS("") + + var reader io.Reader + if reader, err = dw.Download("swagger", "", ""); err != nil { + return + } + + extFile := dw.GetTargetFile() + + if output == "" { + output = home.GetUserDataDir() + } + if err = os.MkdirAll(filepath.Dir(output), 0755); err != nil { + return + } + + targetFile := filepath.Base(extFile) + fmt.Println("start to save", filepath.Join(output, targetFile)) + if err = downloader.WriteTo(reader, output, targetFile); err == nil { + err = decompressData(filepath.Join(output, targetFile)) + } + return +} + +func SwaggersHandler(w http.ResponseWriter, _ *http.Request, + _ map[string]string) { + swaggers := GetSwaggerList() + if data, err := json.Marshal(swaggers); err == nil { + _, _ = w.Write(data) + } else { + w.WriteHeader(http.StatusInternalServerError) + } +} + +func GetSwaggerList() (swaggers []string) { + dataDir := home.GetUserDataDir() + _ = filepath.WalkDir(dataDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + + if !d.IsDir() && filepath.Ext(path) == ".json" { + swaggers = append(swaggers, filepath.Base(path)) + } + return nil + }) + return +} + +func decompressData(dataFile string) (err error) { + var file *os.File + file, err = os.Open(dataFile) + if err != nil { + return + } + defer file.Close() + + var gzipReader *gzip.Reader + gzipReader, err = gzip.NewReader(file) + if err != nil { + return + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break // 退出循环 + } + if err != nil { + panic(err) + } + + destPath := filepath.Join(filepath.Dir(dataFile), filepath.Base(header.Name)) + + switch header.Typeflag { + case tar.TypeReg: + destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY, os.FileMode(header.Mode)) + if err != nil { + panic(err) + } + defer destFile.Close() + + if _, err := io.Copy(destFile, tarReader); err != nil { + panic(err) + } + default: + fmt.Printf("Skipping entry type %c: %s\n", header.Typeflag, header.Name) + } + } + return +} diff --git a/pkg/apispec/swagger.go b/pkg/apispec/swagger.go index 2b30536..958a875 100644 --- a/pkg/apispec/swagger.go +++ b/pkg/apispec/swagger.go @@ -18,8 +18,11 @@ package apispec import ( "github.com/go-openapi/spec" + "github.com/linuxsuren/api-testing/pkg/util/home" "io" "net/http" + "os" + "path/filepath" "regexp" "strings" ) @@ -123,6 +126,12 @@ func ParseToSwagger(data []byte) (swagger *spec.Swagger, err error) { } func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) { + if strings.HasPrefix(swaggerURL, "atest://") { + swaggerURL = strings.ReplaceAll(swaggerURL, "atest://", "") + swagger, err = ParseFileToSwagger(filepath.Join(home.GetUserDataDir(), swaggerURL)) + return + } + var resp *http.Response if resp, err = http.Get(swaggerURL); err == nil && resp != nil && resp.StatusCode == http.StatusOK { swagger, err = ParseStreamToSwagger(resp.Body) @@ -130,6 +139,14 @@ func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) { return } +func ParseFileToSwagger(dataFile string) (swagger *spec.Swagger, err error) { + var data []byte + if data, err = os.ReadFile(dataFile); err == nil { + swagger, err = ParseToSwagger(data) + } + return +} + func ParseStreamToSwagger(stream io.Reader) (swagger *spec.Swagger, err error) { var data []byte if data, err = io.ReadAll(stream); err == nil { diff --git a/pkg/downloader/extension.go b/pkg/downloader/extension.go index a696f2e..62b30b6 100644 --- a/pkg/downloader/extension.go +++ b/pkg/downloader/extension.go @@ -46,10 +46,15 @@ func NewStoreDownloader() PlatformAwareOCIDownloader { func (d *extensionDownloader) Download(name, tag, _ string) (reader io.Reader, err error) { name = strings.TrimPrefix(name, fmt.Sprintf("atest-%s-", d.kind)) - d.extFile = fmt.Sprintf("atest-%s-%s_%s_%s/atest-%s-%s", d.kind, name, d.os, d.arch, d.kind, name) - if d.os == "windows" { - d.extFile = fmt.Sprintf("%s.exe", d.extFile) + if d.os == "" { + d.extFile = fmt.Sprintf("atest-%s-%s.tar.gz", d.kind, name) + } else { + d.extFile = fmt.Sprintf("atest-%s-%s_%s_%s/atest-%s-%s", d.kind, name, d.os, d.arch, d.kind, name) + if d.os == "windows" { + d.extFile = fmt.Sprintf("%s.exe", d.extFile) + } } + image := fmt.Sprintf("%s/atest-ext-%s-%s", d.imagePrefix, d.kind, name) reader, err = d.OCIDownloader.Download(image, tag, d.extFile) return diff --git a/pkg/downloader/oci.go b/pkg/downloader/oci.go index 487ca70..304f8cf 100644 --- a/pkg/downloader/oci.go +++ b/pkg/downloader/oci.go @@ -72,6 +72,8 @@ func (d *defaultOCIDownloader) WithBasicAuth(username string, password string) { } func (d *defaultOCIDownloader) Download(image, tag, file string) (reader io.Reader, err error) { + fmt.Println("start to download", image) + if d.registry == "" { d.registry = getRegistry(image) } diff --git a/pkg/util/home/common.go b/pkg/util/home/common.go index 15962f3..ab6608d 100644 --- a/pkg/util/home/common.go +++ b/pkg/util/home/common.go @@ -1,5 +1,5 @@ /* -Copyright 2024 API Testing Authors. +Copyright 2024-2025 API Testing Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33,6 +33,10 @@ func GetUserBinDir() string { return filepath.Join(GetUserConfigDir(), "bin") } +func GetUserDataDir() string { + return filepath.Join(GetUserConfigDir(), "data") +} + func GetExtensionSocketPath(name string) string { return filepath.Join(GetUserConfigDir(), fmt.Sprintf("%s.sock", name)) } diff --git a/pkg/util/home/common_test.go b/pkg/util/home/common_test.go new file mode 100644 index 0000000..d002294 --- /dev/null +++ b/pkg/util/home/common_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2025 API Testing Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package home + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetUserBinDir(t *testing.T) { + assert.Contains(t, GetUserConfigDir(), "atest") + assert.Contains(t, GetUserBinDir(), "bin") + assert.Contains(t, GetUserDataDir(), "data") + assert.Contains(t, GetExtensionSocketPath("fake"), "fake.sock") +}