support to load swaggers from an extension

cmd/extension.go

@@ -17,75 +17,75 @@ limitations under the License.
 package cmd
 
 import (
-    "fmt"
+	"fmt"
-    "io"
+	"io"
-    "path/filepath"
+	"path/filepath"
-    "runtime"
+	"runtime"
-    "time"
+	"time"
 
-    "github.com/linuxsuren/api-testing/pkg/downloader"
+	"github.com/linuxsuren/api-testing/pkg/downloader"
-    "github.com/spf13/cobra"
+	"github.com/spf13/cobra"
 )
 
 type extensionOption struct {
-    ociDownloader downloader.PlatformAwareOCIDownloader
+	ociDownloader downloader.PlatformAwareOCIDownloader
-    output        string
+	output        string
-    registry      string
+	registry      string
-    kind          string
+	kind          string
-    tag           string
+	tag           string
-    os            string
+	os            string
-    arch          string
+	arch          string
-    timeout       time.Duration
+	timeout       time.Duration
-    imagePrefix   string
+	imagePrefix   string
 }
 
 func createExtensionCommand(ociDownloader downloader.PlatformAwareOCIDownloader) (c *cobra.Command) {
-    opt := &extensionOption{
+	opt := &extensionOption{
-        ociDownloader: ociDownloader,
+		ociDownloader: ociDownloader,
-    }
+	}
-    c = &cobra.Command{
+	c = &cobra.Command{
-        Use:   "extension",
+		Use:   "extension",
-        Short: "Download extension binary files",
+		Short: "Download extension binary files",
-        Long:  "Download the store extension files",
+		Long:  "Download the store extension files",
-        Args:  cobra.MinimumNArgs(1),
+		Args:  cobra.MinimumNArgs(1),
-        RunE:  opt.runE,
+		RunE:  opt.runE,
-    }
+	}
-    flags := c.Flags()
+	flags := c.Flags()
-    flags.StringVarP(&opt.output, "output", "", ".", "The target directory")
+	flags.StringVarP(&opt.output, "output", "", ".", "The target directory")
-    flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty")
+	flags.StringVarP(&opt.tag, "tag", "", "", "The extension image tag, try to find the latest one if this is empty")
-    flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io")
+	flags.StringVarP(&opt.registry, "registry", "", "", "The target extension image registry, supported: docker.io, ghcr.io")
-    flags.StringVarP(&opt.kind, "kind", "", "store", "The extension kind")
+	flags.StringVarP(&opt.kind, "kind", "", "store", "The extension kind")
-    flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS")
+	flags.StringVarP(&opt.os, "os", "", runtime.GOOS, "The OS")
-    flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture")
+	flags.StringVarP(&opt.arch, "arch", "", runtime.GOARCH, "The architecture")
-    flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading")
+	flags.DurationVarP(&opt.timeout, "timeout", "", time.Minute, "The timeout of downloading")
-    flags.StringVarP(&opt.imagePrefix, "image-prefix", "", "linuxsuren", "The prefix for the image address")
+	flags.StringVarP(&opt.imagePrefix, "image-prefix", "", "linuxsuren", "The prefix for the image address")
-    return
+	return
 }
 
 func (o *extensionOption) runE(cmd *cobra.Command, args []string) (err error) {
-    o.ociDownloader.WithOS(o.os)
+	o.ociDownloader.WithOS(o.os)
-    o.ociDownloader.WithArch(o.arch)
+	o.ociDownloader.WithArch(o.arch)
-    o.ociDownloader.WithRegistry(o.registry)
+	o.ociDownloader.WithRegistry(o.registry)
-    o.ociDownloader.WithImagePrefix(o.imagePrefix)
+	o.ociDownloader.WithImagePrefix(o.imagePrefix)
-    o.ociDownloader.WithTimeout(o.timeout)
+	o.ociDownloader.WithTimeout(o.timeout)
-    o.ociDownloader.WithKind(o.kind)
+	o.ociDownloader.WithKind(o.kind)
-    o.ociDownloader.WithContext(cmd.Context())
+	o.ociDownloader.WithContext(cmd.Context())
 
-    for _, arg := range args {
+	for _, arg := range args {
-        var reader io.Reader
+		var reader io.Reader
-        if reader, err = o.ociDownloader.Download(arg, o.tag, ""); err != nil {
+		if reader, err = o.ociDownloader.Download(arg, o.tag, ""); err != nil {
-            return
+			return
-        } else if reader == nil {
+		} else if reader == nil {
-            err = fmt.Errorf("cannot find %s", arg)
+			err = fmt.Errorf("cannot find %s", arg)
-            return
+			return
-        }
+		}
-        extFile := o.ociDownloader.GetTargetFile()
+		extFile := o.ociDownloader.GetTargetFile()
-        cmd.Println("found target file", extFile)
+		cmd.Println("found target file", extFile)
 
-        targetFile := filepath.Base(extFile)
+		targetFile := filepath.Base(extFile)
-        if err = downloader.WriteTo(reader, o.output, targetFile); err == nil {
+		if err = downloader.WriteTo(reader, o.output, targetFile); err == nil {
-            cmd.Println("downloaded", targetFile)
+			cmd.Println("downloaded", targetFile)
-        }
+		}
-    }
+	}
-    return
+	return
 }

cmd/server.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/linuxsuren/api-testing/pkg/apispec"
 	"net"
 	"net/http"
 	"os"
@@ -302,6 +303,15 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		_ = o.httpServer.Shutdown(ctx)
 	}()
 
+	go func() {
+		err := apispec.DownloadSwaggerData("", extDownloader)
+		if err != nil {
+			fmt.Println("failed to download swagger data", err)
+		} else {
+			fmt.Println("success to download swagger data")
+		}
+	}()
+
 	mux := runtime.NewServeMux(runtime.WithMetadata(server.MetadataStoreFunc),
 		runtime.WithMarshalerOption("application/json+pretty", &runtime.JSONPb{
 			MarshalOptions: protojson.MarshalOptions{
@@ -342,6 +352,7 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) {
 		mux.HandlePath(http.MethodGet, "/get", o.getAtestBinary)
 		mux.HandlePath(http.MethodPost, "/runner/{suite}/{case}", service.WebRunnerHandler)
 		mux.HandlePath(http.MethodGet, "/api/v1/sbom", service.SBomHandler)
+		mux.HandlePath(http.MethodGet, "/api/v1/swaggers", apispec.SwaggersHandler)
 
 		postRequestProxyFunc := postRequestProxy(o.skyWalking)
 		mux.HandlePath(http.MethodPost, "/browser/{app}", postRequestProxyFunc)

console/atest-ui/src/views/TestSuite.vue

@@ -4,7 +4,7 @@ import { reactive, ref, watch } from 'vue'
 import { Edit, CopyDocument, Delete } from '@element-plus/icons-vue'
 import type { FormInstance, FormRules } from 'element-plus'
 import type { Suite, TestCase, Pair } from './types'
-import { NewSuggestedAPIsQuery, GetHTTPMethods } from './types'
+import { NewSuggestedAPIsQuery, GetHTTPMethods, SwaggerSuggestion } from './types'
 import EditButton from '../components/EditButton.vue'
 import { Cache } from './cache'
 import { useI18n } from 'vue-i18n'
@@ -20,6 +20,7 @@ const props = defineProps({
 })
 const emit = defineEmits(['updated'])
 let querySuggestedAPIs = NewSuggestedAPIsQuery(Cache.GetCurrentStore().name, props.name!)
+const querySwaggers = SwaggerSuggestion()
 
 const suite = ref({
   name: '',
@@ -325,7 +326,10 @@ const renameTestSuite = (name: string) => {
             </el-select>
           </td>
           <td>
-            <el-input class="mx-1" v-model="suite.spec.url" placeholder="API Spec URL"></el-input>
+              <el-autocomplete
+                  v-model="suite.spec.url"
+                  :fetch-suggestions="querySwaggers"
+              />
           </td>
         </tr>
       </table>

console/atest-ui/src/views/net.ts

@@ -620,6 +620,12 @@ function GetSuggestedAPIs(name: string,
     .then(callback)
 }
 
+function GetSwaggers(callback: (d: any) => void) {
+    fetch(`/api/v1/swaggers`, {})
+        .then(DefaultResponseProcess)
+        .then(callback)
+}
+
 function ReloadMockServer(config: any) {
   const requestOptions = {
     method: 'POST',
@@ -812,7 +818,7 @@ export const API = {
   CreateOrUpdateStore, GetStores, DeleteStore, VerifyStore,
   FunctionsQuery,
   GetSecrets, DeleteSecret, CreateOrUpdateSecret,
-  GetSuggestedAPIs,
+  GetSuggestedAPIs, GetSwaggers,
   ReloadMockServer, GetMockConfig, SBOM, DataQuery,
   getToken
 }

console/atest-ui/src/views/types.ts

@@ -102,6 +102,28 @@ export function NewSuggestedAPIsQuery(store: string, suite: string) {
     })
   }
 }
+
+interface SwaggerItem {
+    value: string
+}
+
+export function SwaggerSuggestion() {
+    return function (queryString: string, cb: (arg: any) => void) {
+        API.GetSwaggers((e) => {
+            var swaggers = [] as SwaggerItem[]
+            e.forEach((item: string) => {
+                swaggers.push({
+                    "value": `atest://${item}`
+                })
+            })
+
+            const results = queryString ? swaggers.filter((item: SwaggerItem) => {
+                return item.value.toLowerCase().indexOf(queryString.toLowerCase()) != -1
+            }) : swaggers
+            cb(results.slice(0, 10))
+        })
+    }
+}
 export function CreateFilter(queryString: string) {
   return (v: Pair) => {
     return v.value.toLowerCase().indexOf(queryString.toLowerCase()) !== -1

pkg/apispec/remote_swagger.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2025 API Testing Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apispec
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"github.com/linuxsuren/api-testing/pkg/downloader"
+	"github.com/linuxsuren/api-testing/pkg/util/home"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+func DownloadSwaggerData(output string, dw downloader.PlatformAwareOCIDownloader) (err error) {
+	dw.WithKind("data")
+	dw.WithOS("")
+
+	var reader io.Reader
+	if reader, err = dw.Download("swagger", "", ""); err != nil {
+		return
+	}
+
+	extFile := dw.GetTargetFile()
+
+	if output == "" {
+		output = home.GetUserDataDir()
+	}
+	if err = os.MkdirAll(filepath.Dir(output), 0755); err != nil {
+		return
+	}
+
+	targetFile := filepath.Base(extFile)
+	fmt.Println("start to save", filepath.Join(output, targetFile))
+	if err = downloader.WriteTo(reader, output, targetFile); err == nil {
+		err = decompressData(filepath.Join(output, targetFile))
+	}
+	return
+}
+
+func SwaggersHandler(w http.ResponseWriter, _ *http.Request,
+	_ map[string]string) {
+	swaggers := GetSwaggerList()
+	if data, err := json.Marshal(swaggers); err == nil {
+		_, _ = w.Write(data)
+	} else {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func GetSwaggerList() (swaggers []string) {
+	dataDir := home.GetUserDataDir()
+	_ = filepath.WalkDir(dataDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !d.IsDir() && filepath.Ext(path) == ".json" {
+			swaggers = append(swaggers, filepath.Base(path))
+		}
+		return nil
+	})
+	return
+}
+
+func decompressData(dataFile string) (err error) {
+	var file *os.File
+	file, err = os.Open(dataFile)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	var gzipReader *gzip.Reader
+	gzipReader, err = gzip.NewReader(file)
+	if err != nil {
+		return
+	}
+	defer gzipReader.Close()
+
+	tarReader := tar.NewReader(gzipReader)
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break // 退出循环
+		}
+		if err != nil {
+			panic(err)
+		}
+
+		destPath := filepath.Join(filepath.Dir(dataFile), filepath.Base(header.Name))
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY, os.FileMode(header.Mode))
+			if err != nil {
+				panic(err)
+			}
+			defer destFile.Close()
+
+			if _, err := io.Copy(destFile, tarReader); err != nil {
+				panic(err)
+			}
+		default:
+			fmt.Printf("Skipping entry type %c: %s\n", header.Typeflag, header.Name)
+		}
+	}
+	return
+}

pkg/apispec/swagger.go

@@ -18,8 +18,11 @@ package apispec
 
 import (
 	"github.com/go-openapi/spec"
+	"github.com/linuxsuren/api-testing/pkg/util/home"
 	"io"
 	"net/http"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -123,6 +126,12 @@ func ParseToSwagger(data []byte) (swagger *spec.Swagger, err error) {
 }
 
 func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) {
+	if strings.HasPrefix(swaggerURL, "atest://") {
+		swaggerURL = strings.ReplaceAll(swaggerURL, "atest://", "")
+		swagger, err = ParseFileToSwagger(filepath.Join(home.GetUserDataDir(), swaggerURL))
+		return
+	}
+
 	var resp *http.Response
 	if resp, err = http.Get(swaggerURL); err == nil && resp != nil && resp.StatusCode == http.StatusOK {
 		swagger, err = ParseStreamToSwagger(resp.Body)
@@ -130,6 +139,14 @@ func ParseURLToSwagger(swaggerURL string) (swagger *spec.Swagger, err error) {
 	return
 }
 
+func ParseFileToSwagger(dataFile string) (swagger *spec.Swagger, err error) {
+	var data []byte
+	if data, err = os.ReadFile(dataFile); err == nil {
+		swagger, err = ParseToSwagger(data)
+	}
+	return
+}
+
 func ParseStreamToSwagger(stream io.Reader) (swagger *spec.Swagger, err error) {
 	var data []byte
 	if data, err = io.ReadAll(stream); err == nil {

pkg/downloader/extension.go

@@ -46,10 +46,15 @@ func NewStoreDownloader() PlatformAwareOCIDownloader {
 
 func (d *extensionDownloader) Download(name, tag, _ string) (reader io.Reader, err error) {
 	name = strings.TrimPrefix(name, fmt.Sprintf("atest-%s-", d.kind))
-	d.extFile = fmt.Sprintf("atest-%s-%s_%s_%s/atest-%s-%s", d.kind, name, d.os, d.arch, d.kind, name)
+	if d.os == "" {
-	if d.os == "windows" {
+		d.extFile = fmt.Sprintf("atest-%s-%s.tar.gz", d.kind, name)
-		d.extFile = fmt.Sprintf("%s.exe", d.extFile)
+	} else {
+		d.extFile = fmt.Sprintf("atest-%s-%s_%s_%s/atest-%s-%s", d.kind, name, d.os, d.arch, d.kind, name)
+		if d.os == "windows" {
+			d.extFile = fmt.Sprintf("%s.exe", d.extFile)
+		}
 	}
+
 	image := fmt.Sprintf("%s/atest-ext-%s-%s", d.imagePrefix, d.kind, name)
 	reader, err = d.OCIDownloader.Download(image, tag, d.extFile)
 	return

pkg/downloader/oci.go

@@ -72,6 +72,8 @@ func (d *defaultOCIDownloader) WithBasicAuth(username string, password string) {
 }
 
 func (d *defaultOCIDownloader) Download(image, tag, file string) (reader io.Reader, err error) {
+	fmt.Println("start to download", image)
+
 	if d.registry == "" {
 		d.registry = getRegistry(image)
 	}

pkg/util/home/common.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2024 API Testing Authors.
+Copyright 2024-2025 API Testing Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,6 +33,10 @@ func GetUserBinDir() string {
 	return filepath.Join(GetUserConfigDir(), "bin")
 }
 
+func GetUserDataDir() string {
+	return filepath.Join(GetUserConfigDir(), "data")
+}
+
 func GetExtensionSocketPath(name string) string {
 	return filepath.Join(GetUserConfigDir(), fmt.Sprintf("%s.sock", name))
 }

pkg/util/home/common_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2025 API Testing Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package home
+
+import (
+    "github.com/stretchr/testify/assert"
+    "testing"
+)
+
+func TestGetUserBinDir(t *testing.T) {
+    assert.Contains(t, GetUserConfigDir(), "atest")
+    assert.Contains(t, GetUserBinDir(), "bin")
+    assert.Contains(t, GetUserDataDir(), "data")
+    assert.Contains(t, GetExtensionSocketPath("fake"), "fake.sock")
+}