From 9b8cee315ecc30e10e9d450f13c4b6d042330c1b Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Tue, 26 Sep 2023 07:29:10 +0800 Subject: [PATCH] feat: support integrating with skywalking (#219) * feat: support integrating with skywalking * add skywalking browser support --- .github/workflows/build.yaml | 4 +- CONTRIBUTION.md | 16 +++ Dockerfile | 30 +++-- Makefile | 5 +- README.md | 1 + cmd/server.go | 20 ++++ cmd/server_test.go | 30 +++++ console/atest-ui/package-lock.json | 27 +++++ console/atest-ui/package.json | 1 + console/atest-ui/src/App.vue | 9 ++ console/atest-ui/vite.config.ts | 8 ++ docs/README.md | 10 +- go.mod | 5 +- go.sum | 119 ++++++++++++++++++- go.work.sum | 179 ++++++++++------------------- main.go | 1 + pkg/server/remote_server.go | 12 +- 17 files changed, 333 insertions(+), 144 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8693831..012b0fa 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -42,7 +42,7 @@ jobs: go-version: 1.18.x - name: API Test run: | - make build copy + TOOLEXEC= make build copy sudo atest service install sudo atest service restart sudo atest service status @@ -108,7 +108,7 @@ jobs: cache-dependency-path: console/atest-ui/package-lock.json - name: Build run: | - make build-embed-ui copy + TOOLEXEC= make build-embed-ui copy sudo atest service install sudo atest service restart - name: Test diff --git a/CONTRIBUTION.md b/CONTRIBUTION.md index ebe3357..9b8ad19 100644 --- a/CONTRIBUTION.md +++ b/CONTRIBUTION.md @@ -24,3 +24,19 @@ Other usage of this: * `/debug/pprof/heap?gc=1` * `/debug/pprof/heap?seconds=10` * `/debug/pprof/goroutine/?debug=0` + +## Skywalking + +```shell +docker run -p 12800:12800 -p 9412:9412 docker.io/apache/skywalking-oap-server:9.0.0 +docker run -p 8080:8080 -e SW_OAP_ADDRESS=http://localhost:12800 -e SW_ZIPKIN_ADDRESS=http://localhost:9412 docker.io/apache/skywalking-ui:9.0.0 + +make build + +export SW_AGENT_NAME=atest +export SW_AGENT_REPORTER_GRPC_BACKEND_SERVICE=10.121.218.184:31065 +export SW_AGENT_PLUGIN_CONFIG_HTTP_SERVER_COLLECT_PARAMETERS=true +export SW_AGENT_METER_COLLECT_INTERVAL=3 +export SW_AGENT_LOG_TYPE=std +./bin/atest server --local-storage 'bin/*.yaml' --http-port 8082 --port 7072 --console-path console/atest-ui/dist/ +``` \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1b4713d..0df83db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,23 @@ +FROM node:20-alpine3.17 AS ui + +WORKDIR /workspace +COPY console/atest-ui . +RUN npm install --ignore-scripts --registry=https://registry.npmmirror.com +RUN npm run build-only + +FROM apache/skywalking-go:0.2.0-go1.18 AS sk + FROM golang:1.18 AS builder ARG VERSION ARG GOPROXY +# it could be -toolexec="skywalking-go-agent" +ARG TOOLEXEC WORKDIR /workspace COPY cmd/ cmd/ COPY pkg/ pkg/ COPY extensions/ extensions/ COPY operator/ operator/ -COPY console/atest-ui atest-ui/ COPY .github/testing/*.yaml sample/ COPY sample/ sample/ COPY docs/ docs/ @@ -19,20 +29,19 @@ COPY main.go main.go COPY README.md README.md COPY LICENSE LICENSE +COPY --from=ui /workspace/dist/index.html cmd/data/index.html +COPY --from=ui /workspace/dist/assets/*.js cmd/data/index.js +COPY --from=ui /workspace/dist/assets/*.css cmd/data/index.css + +COPY --from=sk /usr/local/bin/skywalking-go-agent /usr/local/bin/skywalking-go-agent + RUN GOPROXY=${GOPROXY} go mod download -RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=${VERSION}" -o atest . +RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build ${TOOLEXEC} -a -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=${VERSION}" -o atest . RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-collector extensions/collector/main.go RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-orm extensions/store-orm/main.go RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-s3 extensions/store-s3/main.go RUN GOPROXY=${GOPROXY} CGO_ENABLED=0 go build -ldflags "-w -s" -o atest-store-git extensions/store-git/main.go -FROM node:20-alpine3.17 AS ui - -WORKDIR /workspace -COPY --from=builder /workspace/atest-ui . -RUN npm install --ignore-scripts --registry=https://registry.npmmirror.com -RUN npm run build-only - FROM ubuntu:23.04 LABEL "com.github.actions.name"="API testing" @@ -56,10 +65,9 @@ COPY --from=builder /workspace/README.md /README.md RUN mkdir -p /var/www/data COPY --from=builder /workspace/sample /var/www/sample -COPY --from=ui /workspace/dist /var/www/html RUN apt update -y && \ # required for atest-store-git apt install -y --no-install-recommends ssh-client ca-certificates -CMD ["atest", "server", "--console-path=/var/www/html", "--local-storage=/var/www/sample/*.yaml", "--local-storage=/var/www/data/*.yaml"] +CMD ["atest", "server", "--local-storage=/var/www/sample/*.yaml", "--local-storage=/var/www/data/*.yaml"] diff --git a/Makefile b/Makefile index c5ff78f..8c54895 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ IMG_TOOL?=podman BINARY?=atest +TOOLEXEC?=-toolexec="skywalking-go-agent" build: mkdir -p bin rm -rf bin/atest - go build -o bin/atest main.go + go build ${TOOLEXEC} -a -o bin/atest main.go build-ext-git: CGO_ENABLED=0 go build -ldflags "-w -s" -o bin/atest-store-git extensions/store-git/main.go embed-ui: @@ -17,7 +18,7 @@ clean-embed-ui: git checkout cmd/data/index.js git checkout cmd/data/index.css build-embed-ui: embed-ui - GOOS=${OS} go build -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=$(shell git rev-parse --short HEAD)" -o bin/${BINARY} main.go + GOOS=${OS} go build ${TOOLEXEC} -a -ldflags "-w -s -X github.com/linuxsuren/api-testing/pkg/version.version=$(shell git rev-parse --short HEAD)" -o bin/${BINARY} main.go make clean-embed-ui build-win-embed-ui: BINARY=atest.exe OS=windows make build-embed-ui diff --git a/README.md b/README.md index 92d7b11..2423e24 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ This is a API testing tool. * Multiple storage backends supported(Local, ORM Database, S3, Git, etc) * [HTTP API record](extensions/collector) * Install in mutiple use cases(CLI, Container, Native-Service, Operator, etc) +* Monitoring integration with Prometheus, Skywalking ## Get started diff --git a/cmd/server.go b/cmd/server.go index 955a28a..9691299 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -53,6 +53,7 @@ func createServerCmd(execer fakeruntime.Execer, gRPCServer gRPCServer, httpServe flags.StringVarP(&opt.consolePath, "console-path", "", "", "The path of the console") flags.StringVarP(&opt.configDir, "config-dir", "", os.ExpandEnv("$HOME/.config/atest"), "The config directory") flags.StringVarP(&opt.secretServer, "secret-server", "", "", "The secret server URL") + flags.StringVarP(&opt.skyWalking, "skywalking", "", "", "Push the browser tracing data to the Apache SkyWalking URL") return } @@ -68,6 +69,7 @@ type serverOption struct { consolePath string secretServer string configDir string + skyWalking string } func (o *serverOption) preRunE(cmd *cobra.Command, args []string) (err error) { @@ -159,6 +161,10 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { mux.HandlePath(http.MethodGet, "/healthz", frontEndHandlerWithLocation(o.consolePath)) mux.HandlePath(http.MethodGet, "/get", o.getAtestBinary) + postRequestProxyFunc := postRequestProxy(o.skyWalking) + mux.HandlePath(http.MethodPost, "/browser/{app}", postRequestProxyFunc) + mux.HandlePath(http.MethodPost, "/v3/segments", postRequestProxyFunc) + // Create non-global registry. reg := prometheus.NewRegistry() @@ -180,6 +186,20 @@ func (o *serverOption) runE(cmd *cobra.Command, args []string) (err error) { return } +func postRequestProxy(proxy string) func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) { + if proxy == "" { + return func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) {} + } + + if strings.HasSuffix(proxy, "/") { + proxy = strings.TrimSuffix(proxy, "/") + } + + return func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) { + http.Post(fmt.Sprintf("%s%s", proxy, r.URL.Path), "application/json", r.Body) + } +} + func startPlugins(execer fakeruntime.Execer, kinds *server.StoreKinds) (err error) { const socketPrefix = "unix://" diff --git a/cmd/server_test.go b/cmd/server_test.go index 1f374e4..813b678 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/h2non/gock" "github.com/linuxsuren/api-testing/pkg/server" "github.com/linuxsuren/api-testing/pkg/util" fakeruntime "github.com/linuxsuren/go-fake-runtime" @@ -173,6 +174,35 @@ func TestFrontEndHandlerWithLocation(t *testing.T) { }) } +func TestProxy(t *testing.T) { + t.Run("normal", func(t *testing.T) { + gock.Off() + + gock.New("http://localhost:8080").Post("/api/v1/echo").Reply(http.StatusOK) + gock.New("http://localhost:9090").Post("/api/v1/echo").Reply(http.StatusOK) + + handle := postRequestProxy("http://localhost:9090/") + req, err := http.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/echo", strings.NewReader(`{"message": "hello"}`)) + assert.NoError(t, err) + + resp := newFakeResponseWriter() + handle(resp, req, map[string]string{}) + }) + + t.Run("no proxy", func(t *testing.T) { + gock.Off() + + gock.New("http://localhost:8080").Post("/api/v1/echo").Reply(http.StatusOK) + + handle := postRequestProxy("") + req, err := http.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/echo", strings.NewReader(`{"message": "hello"}`)) + assert.NoError(t, err) + + resp := newFakeResponseWriter() + handle(resp, req, map[string]string{}) + }) +} + type fakeResponseWriter struct { buf *bytes.Buffer header http.Header diff --git a/console/atest-ui/package-lock.json b/console/atest-ui/package-lock.json index a0ac91d..44df02b 100644 --- a/console/atest-ui/package-lock.json +++ b/console/atest-ui/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "element-plus": "^2.3.7", "intro.js": "^7.0.1", + "skywalking-client-js": "^0.10.0", "vue": "^3.3.4", "vue-i18n": "^9.2.2", "vue-json-viewer": "^3.0.4", @@ -8562,6 +8563,11 @@ "@sideway/pinpoint": "^2.0.0" } }, + "node_modules/js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmmirror.com/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==" + }, "node_modules/js-beautify": { "version": "1.14.6", "resolved": "https://registry.npmmirror.com/js-beautify/-/js-beautify-1.14.6.tgz", @@ -10554,6 +10560,14 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "dev": true }, + "node_modules/skywalking-client-js": { + "version": "0.10.0", + "resolved": "https://registry.npmmirror.com/skywalking-client-js/-/skywalking-client-js-0.10.0.tgz", + "integrity": "sha512-WlsusgsfpHeKG6ssinN7DQRuIt5hFmwGTP0QiRepY7CmnrLvLRkyZEEbfdRc/y+WayrQKkWxjPBWozMick+Csg==", + "dependencies": { + "js-base64": "^3.6.0" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", @@ -18760,6 +18774,11 @@ "@sideway/pinpoint": "^2.0.0" } }, + "js-base64": { + "version": "3.7.5", + "resolved": "https://registry.npmmirror.com/js-base64/-/js-base64-3.7.5.tgz", + "integrity": "sha512-3MEt5DTINKqfScXKfJFrRbxkrnk2AxPWGBL/ycjz4dK8iqiSJ06UxD8jh8xuh6p10TX4t2+7FsBYVxxQbMg+qA==" + }, "js-beautify": { "version": "1.14.6", "resolved": "https://registry.npmmirror.com/js-beautify/-/js-beautify-1.14.6.tgz", @@ -20356,6 +20375,14 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "dev": true }, + "skywalking-client-js": { + "version": "0.10.0", + "resolved": "https://registry.npmmirror.com/skywalking-client-js/-/skywalking-client-js-0.10.0.tgz", + "integrity": "sha512-WlsusgsfpHeKG6ssinN7DQRuIt5hFmwGTP0QiRepY7CmnrLvLRkyZEEbfdRc/y+WayrQKkWxjPBWozMick+Csg==", + "requires": { + "js-base64": "^3.6.0" + } + }, "slash": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/slash/-/slash-3.0.0.tgz", diff --git a/console/atest-ui/package.json b/console/atest-ui/package.json index 325ab50..461f94a 100644 --- a/console/atest-ui/package.json +++ b/console/atest-ui/package.json @@ -18,6 +18,7 @@ "dependencies": { "element-plus": "^2.3.7", "intro.js": "^7.0.1", + "skywalking-client-js": "^0.10.0", "vue": "^3.3.4", "vue-i18n": "^9.2.2", "vue-json-viewer": "^3.0.4", diff --git a/console/atest-ui/src/App.vue b/console/atest-ui/src/App.vue index 1853c90..53a0a15 100644 --- a/console/atest-ui/src/App.vue +++ b/console/atest-ui/src/App.vue @@ -13,6 +13,7 @@ import type { Suite } from './types' import { GetLastTestCaseLocation, SetLastTestCaseLocation } from './views/cache' import { DefaultResponseProcess } from './views/net' import { useI18n } from 'vue-i18n' +import ClientMonitor from 'skywalking-client-js'; const { t } = useI18n() @@ -281,6 +282,14 @@ const filterTestCases = (value: string, data: Tree) => { } const viewName = ref('') +watch(viewName, (val) => { + ClientMonitor.setPerformance({ + service: 'atest-ui', + serviceVersion: 'v0.0.1', + pagePath: val, + useFmp: true + }); +})