From 86bbe4e89192f1764cee3bae623d37498f0e2900 Mon Sep 17 00:00:00 2001 From: Rick <1450685+LinuxSuRen@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:24:21 +0800 Subject: [PATCH] test: add k8s e2e test (#270) Co-authored-by: rick --- .dockerignore | 1 + Makefile | 2 +- e2e/Dockerfile | 12 ++++ e2e/compose-k8s.yaml | 69 +++++++++++++++++++ {extensions/e2e => e2e}/compose.yaml | 0 {extensions/e2e => e2e}/entrypoint.sh | 0 {extensions/e2e => e2e}/git.yaml | 14 ++-- e2e/k8s.sh | 14 ++++ e2e/start.sh | 31 +++++++++ .../e2e => e2e}/test-suite-common.yaml | 6 +- extensions/e2e/Dockerfile | 7 -- extensions/e2e/start.sh | 25 ------- helm/api-testing/templates/deployment.yaml | 4 +- 13 files changed, 144 insertions(+), 41 deletions(-) create mode 100644 e2e/Dockerfile create mode 100644 e2e/compose-k8s.yaml rename {extensions/e2e => e2e}/compose.yaml (100%) rename {extensions/e2e => e2e}/entrypoint.sh (100%) rename {extensions/e2e => e2e}/git.yaml (66%) create mode 100644 e2e/k8s.sh create mode 100755 e2e/start.sh rename {extensions/e2e => e2e}/test-suite-common.yaml (95%) delete mode 100644 extensions/e2e/Dockerfile delete mode 100755 extensions/e2e/start.sh diff --git a/.dockerignore b/.dockerignore index a6807cd..b2c7447 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ console/atest-ui/dist .git/ bin/ dist/ +.vscode/ diff --git a/Makefile b/Makefile index 35edd0b..2b3615a 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ test-operator: test-all-backend: test test-collector test-store-orm test-store-s3 test-store-git test-store-etcd test-all: test-all-backend test-ui test-e2e: - cd extensions/e2e && ./start.sh + cd e2e && ./start.sh && ./start.sh compose-k8s.yaml install-precheck: cp .github/pre-commit .git/hooks/pre-commit diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 0000000..5a51f77 --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,12 @@ +FROM ghcr.io/linuxsuren/api-testing:master + +WORKDIR /workspace +COPY e2e/* . +COPY helm/api-testing api-testing +RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +RUN chmod 700 get_helm.sh +RUN ./get_helm.sh +RUN chmod +x entrypoint.sh +RUN chmod +x k8s.sh + +CMD [ "/workspace/entrypoint.sh" ] diff --git a/e2e/compose-k8s.yaml b/e2e/compose-k8s.yaml new file mode 100644 index 0000000..e5b3157 --- /dev/null +++ b/e2e/compose-k8s.yaml @@ -0,0 +1,69 @@ +version: '3.1' +services: + testing: + build: + context: .. + dockerfile: e2e/Dockerfile + command: [ "/workspace/k8s.sh" ] + depends_on: + server: + condition: service_started + links: + - server + - agent + volumes: + - k8s:/root/.kube + + server: + image: "rancher/k3s:${K3S_VERSION:-v1.25.15-k3s2}" + command: server + tmpfs: + - /run + - /var/run + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + privileged: true + restart: always + environment: + - K3S_TOKEN=abcd + - K3S_KUBECONFIG_OUTPUT=/output/config + - K3S_KUBECONFIG_MODE=666 + volumes: + - k8s:/output + # This is just so that we get the kubeconfig file out + # - .:/output + ports: + - 30000:30000 + # - 6443:6443 # Kubernetes API Server + # - 80:80 # Ingress controller port 80 + # - 443:443 # Ingress controller port 443 + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6443"] + interval: 3s + timeout: 30s + retries: 10 + start_period: 3s + + agent: + image: "rancher/k3s:${K3S_VERSION:-v1.25.15-k3s2}" + tmpfs: + - /run + - /var/run + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + privileged: true + restart: always + environment: + - K3S_URL=https://server:6443 + - K3S_TOKEN=abcd + links: + - server + +volumes: + k8s: {} \ No newline at end of file diff --git a/extensions/e2e/compose.yaml b/e2e/compose.yaml similarity index 100% rename from extensions/e2e/compose.yaml rename to e2e/compose.yaml diff --git a/extensions/e2e/entrypoint.sh b/e2e/entrypoint.sh similarity index 100% rename from extensions/e2e/entrypoint.sh rename to e2e/entrypoint.sh diff --git a/extensions/e2e/git.yaml b/e2e/git.yaml similarity index 66% rename from extensions/e2e/git.yaml rename to e2e/git.yaml index 96c204c..050c890 100644 --- a/extensions/e2e/git.yaml +++ b/e2e/git.yaml @@ -3,12 +3,19 @@ # https://docs.gitlab.com/ee/api/api_resources.html name: atest api: | - {{default "http://localhost:8080/server.Runner" (env "SERVER")}} + {{default "http://localhost:8080" (env "SERVER")}}/server.Runner +param: + server: | + {{default "http://localhost:8080" (env "SERVER")}} items: -- name: CreateStore +- name: healthz before: items: - - httpReady("http://localhost:8080/healthz", 2400) + - httpReady("{{.param.server}}/healthz", 6000) + request: + api: | + {{default "http://localhost:8080" (env "SERVER")}}/healthz +- name: CreateStore request: api: /CreateStore method: POST @@ -20,7 +27,6 @@ items: "name": "atest-store-git" } } - - name: listSuite before: items: diff --git a/e2e/k8s.sh b/e2e/k8s.sh new file mode 100644 index 0000000..19e9a51 --- /dev/null +++ b/e2e/k8s.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +sleep 6 +echo "Running k8s.sh" +helm install --kube-apiserver https://server:6443 --kube-token abcd --kube-insecure-skip-tls-verify \ + api-testing ./api-testing \ + --set service.type=NodePort \ + --set service.nodePort=30000 \ + --set persistence.enabled=false \ + --set image.repository=linuxsuren.docker.scarf.sh/linuxsuren/api-testing \ + --set image.tag=master + +SERVER=http://server:30000 atest run -p git.yaml diff --git a/e2e/start.sh b/e2e/start.sh new file mode 100755 index 0000000..0df1b05 --- /dev/null +++ b/e2e/start.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +file=$1 +if [ "$file" == "" ] +then + file=compose.yaml +fi + +docker-compose version +docker-compose -f "$file" up --build -d + +while true +do + docker-compose -f "$file" ps | grep testing + if [ $? -eq 1 ] + then + code=-1 + docker-compose -f "$file" logs | grep e2e-testing + docker-compose -f "$file" logs | grep e2e-testing | grep Usage + if [ $? -eq 1 ] + then + code=0 + echo "successed" + fi + + docker-compose -f "$file" down + set -e + exit $code + fi + sleep 1 +done diff --git a/extensions/e2e/test-suite-common.yaml b/e2e/test-suite-common.yaml similarity index 95% rename from extensions/e2e/test-suite-common.yaml rename to e2e/test-suite-common.yaml index 661cd2b..d94abf1 100644 --- a/extensions/e2e/test-suite-common.yaml +++ b/e2e/test-suite-common.yaml @@ -3,16 +3,18 @@ # https://docs.gitlab.com/ee/api/api_resources.html name: atest api: | - {{default "http://localhost:8080/server.Runner" (env "SERVER")}} + {{default "http://localhost:8080" (env "SERVER")}}/server.Runner param: suiteName: "{{randAlpha 6}}" caseName: "{{randAlpha 6}}" store: "{{randAlpha 3}}" + server: | + {{default "http://localhost:8080" (env "SERVER")}} items: - name: CreateStore before: items: - - httpReady("http://localhost:8080/healthz", 2400) + - httpReady("{{.param.server}}/healthz", 2400) request: api: /CreateStore method: POST diff --git a/extensions/e2e/Dockerfile b/extensions/e2e/Dockerfile deleted file mode 100644 index 8b96756..0000000 --- a/extensions/e2e/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ghcr.io/linuxsuren/api-testing:master - -WORKDIR /workspace -COPY . . -RUN chmod +x entrypoint.sh - -CMD [ "/workspace/entrypoint.sh" ] diff --git a/extensions/e2e/start.sh b/extensions/e2e/start.sh deleted file mode 100755 index 3d129cc..0000000 --- a/extensions/e2e/start.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -docker-compose version -docker-compose up --build -d - -while true -do - docker-compose ps | grep testing - if [ $? -eq 1 ] - then - code=-1 - docker-compose logs | grep e2e-testing - docker-compose logs | grep e2e-testing | grep Usage - if [ $? -eq 1 ] - then - code=0 - echo "successed" - fi - - docker-compose down - set -e - exit $code - fi - sleep 1 -done diff --git a/helm/api-testing/templates/deployment.yaml b/helm/api-testing/templates/deployment.yaml index dd2f1f6..585fd53 100644 --- a/helm/api-testing/templates/deployment.yaml +++ b/helm/api-testing/templates/deployment.yaml @@ -59,11 +59,11 @@ spec: protocol: TCP livenessProbe: httpGet: - path: / + path: /healthz port: http readinessProbe: httpGet: - path: / + path: /healthz port: http volumeMounts: - name: data