feat: add a HTTP request collector as atest extension (#85)

* feat: add a HTTP request collector as atest extension * add unit tests for collector * add unit tests * support to release atest-collector via goreleaser --------- Co-authored-by: Rick <linuxsuren@users.noreply.github.com>
2023-06-07 22:59:47 +08:00 · 2023-06-07 22:59:47 +08:00 · 8da089067c
parent b0b9bf8913
commit 8da089067c
22 changed files with 634 additions and 18 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@ -16,6 +16,7 @@ jobs:
      - name: Test
        run: |
          go test ./... -coverprofile coverage.out
+          make test test-collector
      - name: Report
        if: github.actor == 'linuxsuren'
        env:
--- a/.gitignore
+++ b/.gitignore
@ -1,5 +1,7 @@
 bin/
 .idea/
 coverage.out
+collector-coverage.out
 dist/
 .vscode/launch.json
+sample.yaml
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@ -7,6 +7,7 @@ before:
 builds:
  - env:
      - CGO_ENABLED=0
+    id: atest
    binary: atest
    goos:
      - linux
@ -16,8 +17,31 @@ builds:
      - -w
      - -s
      - -X github.com/linuxsuren/api-testing/pkg/version.version={{.Version}}
+  - env:
+      - CGO_ENABLED=0
+    id: collector
+    binary: atest-collector
+    main: ./extensions/collector/main.go
+    goos:
+      - linux
+      - windows
+      - darwin
+    ldflags:
+      - -w
+      - -s
 archives:
  - name_template: "{{ .Binary }}-{{ .Os }}-{{ .Arch }}"
+    builds:
+      - atest
+    format_overrides:
+      - goos: windows
+        format: zip
+    files:
+      - README.md
+  - name_template: "{{ .Binary }}-{{ .Os }}-{{ .Arch }}"
+    id: collector
+    builds:
+      - collector
    format_overrides:
      - goos: windows
        format: zip
--- a/3
+++ b/3
@ -17,6 +17,9 @@ copy-restart: build
 test:
 	go test ./... -cover -v -coverprofile=coverage.out
 	go tool cover -func=coverage.out
+test-collector:
+	go test github.com/linuxsuren/api-testing/extensions/collector/./... -cover -v -coverprofile=collector-coverage.out
+	go tool cover -func=collector-coverage.out
 grpc:
 	protoc --go_out=. --go_opt=paths=source_relative \
    --go-grpc_out=. --go-grpc_opt=paths=source_relative \
--- a/README.md
+++ b/README.md
@ -16,6 +16,7 @@ This is a API testing tool.
 *   Output reference between TestCase
 *   Run in server mode, and provide the [gRPC endpoint](pkg/server/server.proto)
 *   [VS Code extension](https://github.com/LinuxSuRen/vscode-api-testing) support
+*   [HTTP API record](extensions/collector)

 ## Get started

--- a/extensions/collector/Makefile
+++ b/extensions/collector/Makefile
@ -0,0 +1,3 @@
+test:
+	go test ./... -cover -v -coverprofile=coverage.out
+	go tool cover -func=coverage.out
--- a/extensions/collector/README.md
+++ b/extensions/collector/README.md
@ -0,0 +1,12 @@
+HTTP API record tool.
+
+## Usage
+
+```shell
+atest-collector --filter-path /answer/api/v1
+```
+
+It will start a HTTP proxy server, and set the server address to your browser proxy (such as: [SwitchyOmega](https://github.com/FelisCatus/SwitchyOmega)).
+
+`atest-collector` will record all HTTP requests which has prefix `/answer/api/v1`, and 
+save it to file `sample.yaml` once you close the server.
--- a/extensions/collector/cmd/collect.go
+++ b/extensions/collector/cmd/collect.go
@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"github.com/elazarl/goproxy"
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg"
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg/filter"
+	"github.com/spf13/cobra"
+)
+
+type option struct {
+	port       int
+	filterPath string
+	output     string
+}
+
+// NewRootCmd creates the root command
+func NewRootCmd() (c *cobra.Command) {
+	opt := &option{}
+	c = &cobra.Command{
+		Use:   "atest-collector",
+		Short: "A collector for API testing, it will start a HTTP proxy server",
+		RunE:  opt.runE,
+	}
+	flags := c.Flags()
+	flags.IntVarP(&opt.port, "port", "p", 8080, "The port for the proxy")
+	flags.StringVarP(&opt.filterPath, "filter-path", "", "", "The path prefix for filtering")
+	flags.StringVarP(&opt.output, "output", "o", "sample.yaml", "The output file")
+
+	_ = cobra.MarkFlagRequired(flags, "filter-path")
+	return
+}
+
+type responseFilter struct {
+	urlFilter *filter.URLPathFilter
+	collects  *pkg.Collects
+}
+
+func (f *responseFilter) filter(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
+	contentType := resp.Header.Get("Content-Type")
+	if !strings.Contains(contentType, "application/json") {
+		return resp
+	}
+
+	req := resp.Request
+	if f.urlFilter.Filter(req.URL) {
+		f.collects.Add(req.Clone(context.TODO()))
+	}
+	return resp
+}
+
+func (o *option) runE(cmd *cobra.Command, args []string) (err error) {
+	urlFilter := &filter.URLPathFilter{PathPrefix: o.filterPath}
+	collects := pkg.NewCollects()
+	responseFilter := &responseFilter{urlFilter: urlFilter, collects: collects}
+
+	proxy := goproxy.NewProxyHttpServer()
+	proxy.Verbose = true
+	proxy.OnResponse().DoFunc(responseFilter.filter)
+
+	exporter := pkg.NewSampleExporter()
+	collects.AddEvent(exporter.Add)
+
+	srv := &http.Server{
+		Addr:    fmt.Sprintf(":%d", o.port),
+		Handler: proxy,
+	}
+
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-sig
+		collects.Stop()
+		_ = srv.Shutdown(context.Background())
+	}()
+
+	cmd.Println("Starting the proxy server with port", o.port)
+	_ = srv.ListenAndServe()
+	var data string
+	if data, err = exporter.Export(); err == nil {
+		err = os.WriteFile(o.output, []byte(data), 0644)
+	}
+	return
+}
--- a/extensions/collector/cmd/collect_test.go
+++ b/extensions/collector/cmd/collect_test.go
@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg"
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg/filter"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewRootCmd(t *testing.T) {
+	c := NewRootCmd()
+	assert.NotNil(t, c)
+	assert.Equal(t, "atest-collector", c.Use)
+}
+
+func TestResponseFilter(t *testing.T) {
+	resp := &http.Response{
+		Header: http.Header{
+			"Content-Type": []string{"application/json; charset=utf-8"},
+		},
+		Request: &http.Request{
+			URL: &url.URL{},
+		},
+	}
+	emptyResp := &http.Response{}
+
+	filter := &responseFilter{
+		urlFilter: &filter.URLPathFilter{},
+		collects:  pkg.NewCollects(),
+	}
+	filter.filter(emptyResp, nil)
+	filter.filter(resp, nil)
+}
--- a/extensions/collector/go.mod
+++ b/extensions/collector/go.mod
@ -0,0 +1,38 @@
+module github.com/linuxsuren/api-testing/extensions/collector
+
+go 1.19
+
+require (
+	github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819
+	github.com/linuxsuren/api-testing v0.0.11
+	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.8.4
+	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.0 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/mitchellh/copystructure v1.0.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	golang.org/x/crypto v0.3.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/linuxsuren/api-testing => ../../.
--- a/extensions/collector/go.sum
+++ b/extensions/collector/go.sum
@ -0,0 +1,96 @@
+github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
+github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
+github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
+github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
+github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/extensions/collector/main.go
+++ b/extensions/collector/main.go
@ -0,0 +1,13 @@
+package main
+
+import (
+	"os"
+
+	"github.com/linuxsuren/api-testing/extensions/collector/cmd"
+)
+
+func main() {
+	if err := cmd.NewRootCmd().Execute(); err != nil {
+		os.Exit(1)
+	}
+}
--- a/extensions/collector/pkg/collector.go
+++ b/extensions/collector/pkg/collector.go
@ -0,0 +1,73 @@
+package pkg
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+)
+
+// Collects is a HTTP request collector
+type Collects struct {
+	once       sync.Once
+	signal     chan string
+	stopSignal chan struct{}
+	keys       map[string]*http.Request
+	requests   []*http.Request
+	events     []EventHandle
+}
+
+// NewCollects creates an instance of Collector
+func NewCollects() *Collects {
+	return &Collects{
+		once:       sync.Once{},
+		signal:     make(chan string, 5),
+		stopSignal: make(chan struct{}, 1),
+		keys:       make(map[string]*http.Request),
+	}
+}
+
+// Add adds a HTTP request
+func (c *Collects) Add(req *http.Request) {
+	key := fmt.Sprintf("%s-%s", req.Method, req.URL.String())
+	if _, ok := c.keys[key]; !ok {
+		c.keys[key] = req
+		c.requests = append(c.requests, req)
+		c.signal <- key
+	}
+}
+
+// EventHandle is the collect event handle
+type EventHandle func(r *http.Request)
+
+// AddEvent adds new event handle
+func (c *Collects) AddEvent(e EventHandle) {
+	c.events = append(c.events, e)
+	c.handleEvents()
+}
+
+// Stop stops the collector
+func (c *Collects) Stop() {
+	c.stopSignal <- struct{}{}
+}
+
+func (c *Collects) handleEvents() {
+	fmt.Println("handle events")
+	c.once.Do(func() {
+		go func() {
+			fmt.Println("start handle events")
+			for {
+				select {
+				case key := <-c.signal:
+					fmt.Println("receive signal", key)
+					for _, e := range c.events {
+						fmt.Println("handle event", key, e)
+						e(c.keys[key])
+					}
+				case <-c.stopSignal:
+					fmt.Println("stop")
+					return
+				}
+			}
+		}()
+	})
+}
--- a/extensions/collector/pkg/collector_test.go
+++ b/extensions/collector/pkg/collector_test.go
@ -0,0 +1,33 @@
+package pkg_test
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCollector(t *testing.T) {
+	defaultReq, _ := http.NewRequest(http.MethodGet, "http://foo.com", nil)
+
+	tests := []struct {
+		name    string
+		Request *http.Request
+	}{{
+		name:    "normal",
+		Request: defaultReq,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			collects := pkg.NewCollects()
+			collects.AddEvent(func(r *http.Request) {
+				assert.Equal(t, tt.Request, r)
+			})
+			for i := 0; i < 10; i++ {
+				collects.Add(tt.Request)
+			}
+			collects.Stop()
+		})
+	}
+}
--- a/extensions/collector/pkg/exporter.go
+++ b/extensions/collector/pkg/exporter.go
@ -0,0 +1,84 @@
+package pkg
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	atestpkg "github.com/linuxsuren/api-testing/pkg/testing"
+	"gopkg.in/yaml.v2"
+)
+
+// SampleExporter is a sample exporter
+type SampleExporter struct {
+	TestSuite atestpkg.TestSuite
+}
+
+// NewSampleExporter creates a new exporter
+func NewSampleExporter() *SampleExporter {
+	return &SampleExporter{
+		TestSuite: atestpkg.TestSuite{
+			Name: "sample",
+		},
+	}
+}
+
+// Add adds a request to the exporter
+func (e *SampleExporter) Add(r *http.Request) {
+
+	fmt.Println("receive", r.URL.Path)
+	req := atestpkg.Request{
+		API:    r.URL.String(),
+		Method: r.Method,
+		Header: map[string]string{},
+	}
+
+	if body := r.Body; body != nil {
+		if data, err := io.ReadAll(body); err == nil {
+			req.Body = string(data)
+		}
+	}
+
+	testCase := atestpkg.TestCase{
+		Request: req,
+		Expect: atestpkg.Response{
+			StatusCode: http.StatusOK,
+		},
+	}
+
+	specs := strings.Split(r.URL.Path, "/")
+	if len(specs) > 0 {
+		testCase.Name = specs[len(specs)-1]
+	}
+
+	if val := r.Header.Get("Content-Type"); val != "" {
+		req.Header["Content-Type"] = val
+	}
+	if val := r.Header.Get("Authorization"); val != "" {
+		req.Header["Authorization"] = val
+	}
+
+	e.TestSuite.Items = append(e.TestSuite.Items, testCase)
+}
+
+var prefix = `#!api-testing
+# yaml-language-server: $schema=https://gitee.com/linuxsuren/api-testing/raw/master/sample/api-testing-schema.json
+`
+
+// Export exports the test suite
+func (e *SampleExporter) Export() (string, error) {
+	marker := map[string]int{}
+
+	for i, item := range e.TestSuite.Items {
+		if _, ok := marker[item.Name]; ok {
+			marker[item.Name]++
+			e.TestSuite.Items[i].Name = fmt.Sprintf("%s-%d", item.Name, marker[item.Name])
+		} else {
+			marker[item.Name] = 0
+		}
+	}
+
+	data, err := yaml.Marshal(e.TestSuite)
+	return prefix + string(data), err
+}
--- a/extensions/collector/pkg/exporter_test.go
+++ b/extensions/collector/pkg/exporter_test.go
@ -0,0 +1,40 @@
+package pkg_test
+
+import (
+	"bytes"
+	"net/http"
+	"testing"
+
+	_ "embed"
+
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSampleExporter(t *testing.T) {
+	exporter := pkg.NewSampleExporter()
+	assert.Equal(t, "sample", exporter.TestSuite.Name)
+
+	request, err := newRequest()
+	assert.NoError(t, err)
+	exporter.Add(request)
+
+	request, err = newRequest()
+	exporter.Add(request)
+
+	var result string
+	result, err = exporter.Export()
+	assert.NoError(t, err)
+	assert.Equal(t, sampleSuite, result)
+}
+
+func newRequest() (request *http.Request, err error) {
+	request, err = http.NewRequest(http.MethodGet, "http://foo/api/v1",
+		bytes.NewBuffer([]byte("hello")))
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("Authorization", "Bearer token")
+	return
+}
+
+//go:embed testdata/sample_suite.yaml
+var sampleSuite string
--- a/extensions/collector/pkg/filter/url_filter.go
+++ b/extensions/collector/pkg/filter/url_filter.go
@ -0,0 +1,23 @@
+package filter
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// URLFilter is the interface of URL filter
+type URLFilter interface {
+	Filter(targetURL *url.URL) bool
+}
+
+// URLPathFilter filters the URL with path
+type URLPathFilter struct {
+	PathPrefix string
+}
+
+// Filter implements the URLFilter
+func (f *URLPathFilter) Filter(targetURL *url.URL) bool {
+	fmt.Println(targetURL.Path, f.PathPrefix)
+	return strings.HasPrefix(targetURL.Path, f.PathPrefix)
+}
--- a/extensions/collector/pkg/filter/url_filter_test.go
+++ b/extensions/collector/pkg/filter/url_filter_test.go
@ -0,0 +1,14 @@
+package filter_test
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/linuxsuren/api-testing/extensions/collector/pkg/filter"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestURLPathFilter(t *testing.T) {
+	urlFilter := &filter.URLPathFilter{PathPrefix: "/api"}
+	assert.True(t, urlFilter.Filter(&url.URL{Path: "/api/v1"}))
+}
--- a/extensions/collector/pkg/testdata/sample_suite.yaml
+++ b/extensions/collector/pkg/testdata/sample_suite.yaml
@ -0,0 +1,24 @@
+#!api-testing
+# yaml-language-server: $schema=https://gitee.com/linuxsuren/api-testing/raw/master/sample/api-testing-schema.json
+name: sample
+items:
+- name: v1
+  request:
+    api: http://foo/api/v1
+    method: GET
+    header:
+      Authorization: Bearer token
+      Content-Type: application/json
+    body: hello
+  expect:
+    statusCode: 200
+- name: v1-1
+  request:
+    api: http://foo/api/v1
+    method: GET
+    header:
+      Authorization: Bearer token
+      Content-Type: application/json
+    body: hello
+  expect:
+    statusCode: 200
--- a/go.work
+++ b/go.work
@ -0,0 +1,4 @@
+go 1.19
+
+use ./
+use ./extensions/collector
--- a/go.work.sum
+++ b/go.work.sum
@ -0,0 +1 @@
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
--- a/pkg/testing/case.go
+++ b/pkg/testing/case.go
@ -2,20 +2,20 @@ package testing

 // TestSuite represents a set of test cases
 type TestSuite struct {
-	Name  string     `yaml:"name" json:"name"`
+	Name  string     `yaml:"name,omitempty" json:"name"`
 	API   string     `yaml:"api,omitempty" json:"api,omitempty"`
 	Items []TestCase `yaml:"items" json:"items"`
 }

 // TestCase represents a test case
 type TestCase struct {
-	Name    string `yaml:"name" json:"name"`
-	Group   string
-	Before  Job      `yaml:"before" json:"before"`
-	After   Job      `yaml:"after" json:"after"`
+	Name    string   `yaml:"name,omitempty" json:"name"`
+	Group   string   `yaml:"group,omitempty" json:"group"`
+	Before  Job      `yaml:"before,omitempty" json:"before"`
+	After   Job      `yaml:"after,omitempty" json:"after"`
 	Request Request  `yaml:"request" json:"request"`
-	Expect  Response `yaml:"expect" json:"expect"`
-	Clean   Clean    `yaml:"clean" json:"-"`
+	Expect  Response `yaml:"expect,omitempty" json:"expect"`
+	Clean   Clean    `yaml:"clean,omitempty" json:"-"`
 }

 // InScope returns true if the test case is in scope with the given items.
@ -41,21 +41,21 @@ type Job struct {
 type Request struct {
 	API          string            `yaml:"api" json:"api"`
 	Method       string            `yaml:"method,omitempty" json:"method,omitempty" jsonschema:"enum=GET,enum=POST,enum=PUT,enum=DELETE"`
-	Query        map[string]string `yaml:"query" json:"query,omitempty"`
-	Header       map[string]string `yaml:"header" json:"header,omitempty"`
-	Form         map[string]string `yaml:"form" json:"form,omitempty"`
-	Body         string            `yaml:"body" json:"body,omitempty"`
-	BodyFromFile string            `yaml:"bodyFromFile" json:"bodyFromFile,omitempty"`
+	Query        map[string]string `yaml:"query,omitempty" json:"query,omitempty"`
+	Header       map[string]string `yaml:"header,omitempty" json:"header,omitempty"`
+	Form         map[string]string `yaml:"form,omitempty" json:"form,omitempty"`
+	Body         string            `yaml:"body,omitempty" json:"body,omitempty"`
+	BodyFromFile string            `yaml:"bodyFromFile,omitempty" json:"bodyFromFile,omitempty"`
 }

 // Response is the expected response
 type Response struct {
-	StatusCode       int                    `yaml:"statusCode" json:"statusCode,omitempty"`
-	Body             string                 `yaml:"body" json:"body,omitempty"`
-	Header           map[string]string      `yaml:"header" json:"header,omitempty"`
-	BodyFieldsExpect map[string]interface{} `yaml:"bodyFieldsExpect" json:"bodyFieldsExpect,omitempty"`
-	Verify           []string               `yaml:"verify" json:"verify,omitempty"`
-	Schema           string                 `yaml:"schema" json:"schema,omitempty"`
+	StatusCode       int                    `yaml:"statusCode,omitempty" json:"statusCode,omitempty"`
+	Body             string                 `yaml:"body,omitempty" json:"body,omitempty"`
+	Header           map[string]string      `yaml:"header,omitempty" json:"header,omitempty"`
+	BodyFieldsExpect map[string]interface{} `yaml:"bodyFieldsExpect,omitempty" json:"bodyFieldsExpect,omitempty"`
+	Verify           []string               `yaml:"verify,omitempty" json:"verify,omitempty"`
+	Schema           string                 `yaml:"schema,omitempty" json:"schema,omitempty"`
 }

 // Clean represents the clean work after testing
				`@ -0,0 +1 @@`
				`github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=`