test: add k8s e2e test (#270)

Co-authored-by: rick <LinuxSuRen@users.noreply.github.com>

.dockerignore

@@ -3,3 +3,4 @@ console/atest-ui/dist
 .git/
 bin/
 dist/
+.vscode/

Makefile

@@ -104,7 +104,7 @@ test-operator:
 test-all-backend: test test-collector test-store-orm test-store-s3 test-store-git test-store-etcd
 test-all: test-all-backend test-ui
 test-e2e:
-	cd extensions/e2e && ./start.sh
+	cd e2e && ./start.sh && ./start.sh compose-k8s.yaml
 install-precheck:
 	cp .github/pre-commit .git/hooks/pre-commit
 

e2e/Dockerfile

@@ -0,0 +1,12 @@
+FROM ghcr.io/linuxsuren/api-testing:master
+
+WORKDIR /workspace
+COPY e2e/* .
+COPY helm/api-testing api-testing
+RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+RUN chmod 700 get_helm.sh
+RUN ./get_helm.sh
+RUN chmod +x entrypoint.sh
+RUN chmod +x k8s.sh
+
+CMD [ "/workspace/entrypoint.sh" ]

e2e/compose-k8s.yaml

@@ -0,0 +1,69 @@
+version: '3.1'
+services:
+  testing:
+    build:
+      context: ..
+      dockerfile: e2e/Dockerfile
+    command: [ "/workspace/k8s.sh" ]
+    depends_on:
+      server:
+        condition: service_started
+    links:
+      - server
+      - agent
+    volumes:
+      - k8s:/root/.kube
+
+  server:
+    image: "rancher/k3s:${K3S_VERSION:-v1.25.15-k3s2}"
+    command: server
+    tmpfs:
+    - /run
+    - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+    - K3S_TOKEN=abcd
+    - K3S_KUBECONFIG_OUTPUT=/output/config
+    - K3S_KUBECONFIG_MODE=666
+    volumes:
+    - k8s:/output
+    # This is just so that we get the kubeconfig file out
+    # - .:/output
+    ports:
+    - 30000:30000
+    # - 6443:6443  # Kubernetes API Server
+    # - 80:80      # Ingress controller port 80
+    # - 443:443    # Ingress controller port 443
+    healthcheck:
+      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/6443"]
+      interval: 3s
+      timeout: 30s
+      retries: 10
+      start_period: 3s
+
+  agent:
+    image: "rancher/k3s:${K3S_VERSION:-v1.25.15-k3s2}"
+    tmpfs:
+    - /run
+    - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+    - K3S_URL=https://server:6443
+    - K3S_TOKEN=abcd
+    links:
+      - server
+
+volumes:
+  k8s: {}

e2e/git.yaml

@@ -3,12 +3,19 @@
 # https://docs.gitlab.com/ee/api/api_resources.html
 name: atest
 api: |
-  {{default "http://localhost:8080/server.Runner" (env "SERVER")}}
+    {{default "http://localhost:8080" (env "SERVER")}}/server.Runner
+param:
+  server: |
+    {{default "http://localhost:8080" (env "SERVER")}}
 items:
-- name: CreateStore
+- name: healthz
   before:
     items:
-      - httpReady("http://localhost:8080/healthz", 2400)
+      - httpReady("{{.param.server}}/healthz", 6000)
+  request:
+    api: |
+      {{default "http://localhost:8080" (env "SERVER")}}/healthz
+- name: CreateStore
   request:
     api: /CreateStore
     method: POST
@@ -20,7 +27,6 @@ items:
           "name": "atest-store-git"
         }
       }
-
 - name: listSuite
   before:
     items:

e2e/k8s.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+sleep 6
+echo "Running k8s.sh"
+helm install --kube-apiserver https://server:6443 --kube-token abcd --kube-insecure-skip-tls-verify \
+    api-testing ./api-testing \
+    --set service.type=NodePort \
+    --set service.nodePort=30000 \
+    --set persistence.enabled=false \
+    --set image.repository=linuxsuren.docker.scarf.sh/linuxsuren/api-testing \
+    --set image.tag=master
+
+SERVER=http://server:30000 atest run -p git.yaml

e2e/start.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+file=$1
+if [ "$file" == "" ]
+then
+    file=compose.yaml
+fi
+
+docker-compose version
+docker-compose -f "$file" up --build -d
+
+while true
+do
+    docker-compose -f "$file" ps | grep testing
+    if [ $? -eq 1 ]
+    then
+        code=-1
+        docker-compose -f "$file" logs | grep e2e-testing
+        docker-compose -f "$file" logs | grep e2e-testing | grep Usage
+        if [ $? -eq 1 ]
+        then
+            code=0
+            echo "successed"
+        fi
+
+        docker-compose -f "$file" down
+        set -e
+        exit $code
+    fi
+    sleep 1
+done

e2e/test-suite-common.yaml

@@ -3,16 +3,18 @@
 # https://docs.gitlab.com/ee/api/api_resources.html
 name: atest
 api: |
-  {{default "http://localhost:8080/server.Runner" (env "SERVER")}}
+  {{default "http://localhost:8080" (env "SERVER")}}/server.Runner
 param:
   suiteName: "{{randAlpha 6}}"
   caseName: "{{randAlpha 6}}"
   store: "{{randAlpha 3}}"
+  server: |
+    {{default "http://localhost:8080" (env "SERVER")}}
 items:
 - name: CreateStore
   before:
     items:
-      - httpReady("http://localhost:8080/healthz", 2400)
+      - httpReady("{{.param.server}}/healthz", 2400)
   request:
     api: /CreateStore
     method: POST

extensions/e2e/Dockerfile

@@ -1,7 +0,0 @@
-FROM ghcr.io/linuxsuren/api-testing:master
-
-WORKDIR /workspace
-COPY . .
-RUN chmod +x entrypoint.sh
-
-CMD [ "/workspace/entrypoint.sh" ]

extensions/e2e/start.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-docker-compose version
-docker-compose up --build -d
-
-while true
-do
-    docker-compose ps | grep testing
-    if [ $? -eq 1 ]
-    then
-        code=-1
-        docker-compose logs | grep e2e-testing
-        docker-compose logs | grep e2e-testing | grep Usage
-        if [ $? -eq 1 ]
-        then
-            code=0
-            echo "successed"
-        fi
-
-        docker-compose down
-        set -e
-        exit $code
-    fi
-    sleep 1
-done

helm/api-testing/templates/deployment.yaml

@@ -59,11 +59,11 @@ spec:
               protocol: TCP
           livenessProbe:
             httpGet:
-              path: /
+              path: /healthz
               port: http
           readinessProbe:
             httpGet:
-              path: /
+              path: /healthz
               port: http
           volumeMounts:
           - name: data