update

2023-08-29 15:32:44 +08:00 · 2023-08-29 15:32:44 +08:00 · 6b3cce57c9
parent 6fb9a0ff7e
commit 6b3cce57c9
189 changed files with 87880 additions and 156 deletions
--- a/group_vars/all
+++ b/group_vars/all
@ -69,8 +69,8 @@ node_taints: ""
 #本地镜像仓库端口
 registry_port: 6550

-BASE_IMAGE_URL: >-
-  {{ groups['registry'][0]}}:{{registry_port}}
+BASE_IMAGE_URL: dev-docker-registry.ccyunchina.com
+  

 #docker insecure registry 如果有多个用逗号分开
 docker_insecure_registry: >-
@ -90,10 +90,10 @@ deploy_offline: true
 deploy_docker_registry: true

 #是否加载镜像
-load_images: true
+load_images: false

 #是否推送镜像
-push_images: true
+push_images: false


 #工具镜像（证书）
@ -104,4 +104,4 @@ gw_tools_image: toyangdon/gw-tools:1.1
 cfssl_cmd: "cd {{ ca_dir }} && export PATH=$PATH:{{bin_dir}} && sh -c"

 #专有云业务镜像仓库
-CLOUD_IMAGE_URL: "{{BASE_IMAGE_URL}}"
+CLOUD_IMAGE_URL: "dev-docker-registry.ccyunchina.com"
--- a/31
+++ b/31
@ -3,22 +3,23 @@ localhost

 #镜像仓库
 [registry]
-10.2.1.212
+10.2.1.99

 #节点配置
 #管理节点 
 [kube-master]
-10.2.1.212
-10.2.1.170
-10.2.1.148
+10.2.1.99
+10.2.1.233
+10.2.1.235

 #计算节点
 [kube-compute-node]

 #存储节点
 [kube-storage-node]
-10.2.1.212
-10.2.1.170
+10.2.1.99
+10.2.1.233
+10.2.1.235

 [kube-node:children]
 kube-compute-node
@ -34,20 +35,16 @@ lb
 #多管理节点时必须配置有代理节点
 #LB_ROLE：master、backup、lb_only，master\backup表示使用keepalived实现haproxy高可用,lb_only表示不使用keepalived
 [lb]
-10.2.1.212
-10.2.1.170
+10.2.1.99
+10.2.1.233

 [elasticsearch]
-10.2.1.212
-10.2.1.170
+10.2.1.99
+10.2.1.233

 [mysql]
-10.2.1.148
-10.2.1.170
-
-#openvpn服务，请另外创建一台虚拟机（4c8g）作为openvpn服务器，该虚拟机可用于暴露私有云内部网络，openvpnExternalIp表示服务器的外部IP，openvpnRouteCIDR表示需求经过vpn路由的ip范围
-[openvpn]
-10.2.1.224  openvpnExternalIp=10.2.1.224 openvpnRouteCIDR=10.2.1.0/16 ansible_ssh_user=root ansible_ssh_pass=Greatwall@123
+10.2.1.235
+10.2.1.233

 # 预留组，后续添加node节点使用
 [new-node]
@ -60,7 +57,7 @@ ansible_python_interpreter=/usr/bin/python3
 MASTER_IP="10.2.1.164"

 #gfs磁盘设备名
-gfs_device=/dev/sdb
+gfs_device=/dev/sda

 zstackServerIP=10.1.1.9
 zstackServerAdminUser=admin
--- a/manifests/calico.yaml
+++ b/manifests/calico.yaml
@ -0,0 +1,455 @@
+# Calico Version v3.3.7
+# https://docs.projectcalico.org/v3.3/releases#v3.3.7
+# This manifest includes the following component versions:
+#   calico/node:v3.3.7
+#   calico/cni:v3.3.7
+#   calico/kube-controllers:v3.3.7
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-kube-controllers
+rules:
+  - apiGroups:
+    - ""
+    - extensions
+    resources:
+      - pods
+      - namespaces
+      - networkpolicies
+      - nodes
+    verbs:
+      - watch
+      - list
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-kube-controllers
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-kube-controllers
+subjects:
+- kind: ServiceAccount
+  name: calico-kube-controllers
+  namespace: kube-system
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-node
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system
+
+
+---
+# This ConfigMap is used to configure a self-hosted Calico installation.
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: calico-config
+  namespace: kube-system
+data:
+  # Configure this with the location of your etcd cluster.
+  etcd_endpoints: "https://10.2.1.99:2379,https://10.2.1.233:2379,https://10.2.1.235:2379"
+
+  # If you're using TLS enabled etcd uncomment the following.
+  # You must also populate the Secret below with these files.
+  etcd_ca: "/calico-secrets/ca.pem"
+  etcd_cert: "/calico-secrets/calico.pem"
+  etcd_key: "/calico-secrets/calico-key.pem"
+  # Configure the Calico backend to use.
+  calico_backend: "bird"
+
+  # Configure the MTU to use
+  veth_mtu: "1440"
+
+  # The CNI network configuration to install on each node.  The special
+  # values in this config will be automatically populated.
+  cni_network_config: |-
+   {
+      "name": "k8s-pod-network",
+      "cniVersion": "0.3.0",
+      "plugins": [ 
+        {
+          "type": "calico",
+          "log_level": "info",
+          "etcd_endpoints": "https://10.2.1.99:2379,https://10.2.1.233:2379,https://10.2.1.235:2379",
+          "etcd_key_file": "/etc/calico/ssl/calico-key.pem",
+          "etcd_cert_file": "/etc/calico/ssl/calico.pem",
+          "etcd_ca_cert_file": "/etc/calico/ssl/ca.pem",
+          "mtu": 1440,
+          "ipam": {
+              "type": "calico-ipam"
+          },
+          "policy": {
+              "type": "k8s"
+          },
+          "kubernetes": {
+              "kubeconfig": "/root/.kube/config"
+          }
+        },
+        {
+          "type": "portmap",
+          "snat": true,
+          "capabilities": {"portMappings": true}
+        }
+      ]
+    }
+
+---
+
+
+# This manifest installs the calico/node container, as well
+# as the Calico CNI plugins and network config on
+# each master and worker node in a Kubernetes cluster.
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+        # This, along with the CriticalAddonsOnly toleration below,
+        # marks the pod as a critical add-on, ensuring it gets
+        # priority scheduling and that its resources are reserved
+        # if it ever gets evicted.
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      hostNetwork: true
+      tolerations:
+        # Make sure calico-node gets scheduled on all nodes.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      serviceAccountName: calico-node
+      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
+      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
+      terminationGracePeriodSeconds: 0
+      containers:
+        # Runs calico/node container on each Kubernetes node.  This
+        # container programs network policy and routes on each
+        # host.
+        - name: calico-node
+          image: dev-docker-registry.ccyunchina.com/calico/node:v3.3.1
+          env:
+            # The location of the Calico etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+            # Set noderef for node controller.
+            - name: CALICO_K8S_NODE_REF
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # Choose the backend to use.
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+            # Cluster type to identify the deployment type
+            - name: CLUSTER_TYPE
+              value: "k8s,bgp"
+            # Auto-detect the BGP IP address.
+            - name: IP
+              value: "autodetect"
+            # Enable IPIP
+            - name: CALICO_IPV4POOL_IPIP
+              value: "cross-subnet"
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: veth_mtu
+            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
+            # chosen from this range. Changing this value after installation will have
+            # no effect. This should fall within `--cluster-cidr`.
+            - name: CALICO_IPV4POOL_CIDR
+              value: "172.20.0.0/16"
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
+            # Disable IPv6 on Kubernetes.
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            # Set Felix logging to "info"
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "info"
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 50m
+              memory: 100Mi
+            limits:
+              cpu: 500m
+              memory: 800Mi
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 9099
+              host: localhost
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            exec:
+              command:
+              - /bin/calico-node
+              - -bird-ready
+              - -felix-ready
+            periodSeconds: 10
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /run/xtables.lock
+              name: xtables-lock
+              readOnly: false
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+            - mountPath: /var/lib/calico
+              name: var-lib-calico
+              readOnly: false
+            - mountPath: /calico-secrets
+              name: etcd-certs
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: dev-docker-registry.ccyunchina.com/calico/cni:v3.3.1
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # The location of the Calico etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            # The CNI network config to install on each node.
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            # CNI MTU Config variable
+            - name: CNI_MTU
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: veth_mtu
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /calico-secrets
+              name: etcd-certs
+      volumes:
+        # Used by calico/node.
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
+        # Used to install CNI.
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/k8s/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        # Mount in the etcd TLS secrets with mode 400.
+        # See https://kubernetes.io/docs/concepts/configuration/secret/
+        - name: etcd-certs
+          hostPath:
+            path: /etc/calico/ssl
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system
+
+---
+
+# This manifest deploys the Calico Kubernetes controllers.
+# See https://github.com/projectcalico/kube-controllers
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system
+  labels:
+    k8s-app: calico-kube-controllers
+spec:
+  # The controllers can only have a single active instance.
+  selector:
+    matchLabels:
+      k8s-app: calico-kube-controllers
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: calico-kube-controllers
+      namespace: kube-system
+      labels:
+        k8s-app: calico-kube-controllers
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      # The controllers must run in the host network namespace so that
+      # it isn't governed by policy that would prevent it from working.
+      hostNetwork: true
+      tolerations:
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      serviceAccountName: calico-kube-controllers
+      containers:
+        - name: calico-kube-controllers
+          image: dev-docker-registry.ccyunchina.com/calico/kube-controllers:v3.3.1
+          env:
+            # The location of the Calico etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+            # Choose which controllers to run.
+            - name: ENABLED_CONTROLLERS
+              value: policy,namespace,serviceaccount,workloadendpoint,node
+          resources:
+            requests:
+              cpu: 100m
+              memory: 40Mi
+            limits:
+              cpu: 150m
+              memory: 100Mi
+          volumeMounts:
+            # Mount in the etcd TLS secrets.
+            - mountPath: /calico-secrets
+              name: etcd-certs
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -r
+      volumes:
+        # Mount in the etcd TLS secrets with mode 400.
+        # See https://kubernetes.io/docs/concepts/configuration/secret/
+        - name: etcd-certs
+          hostPath:
+            path: /etc/calico/ssl
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-kube-controllers
+  namespace: kube-system
--- a/manifests/coredns.yaml
+++ b/manifests/coredns.yaml
@ -0,0 +1,180 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        ready
+        kubernetes cluster.local. in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        reload
+        loadbalance
+    }
+---
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      containers:
+      - name: coredns
+        image: dev-docker-registry.ccyunchina.com/toyangdon/coredns:v1.8.0-arm64
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: 10.68.0.2
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+  - name: metrics
+    port: 9153
+    protocol: TCP
--- a/manifests/es-indices-cleanup.yaml
+++ b/manifests/es-indices-cleanup.yaml
@ -0,0 +1,18 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: es-index-cleaner
+  namespace: kube-system
+spec:
+  # 每天1点3分执行
+  schedule: "3 1 */1 * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: es-index-cleaner
+            image: dev-docker-registry.ccyunchina.com/toyangdon/es-index-cleaner:v0.1
+          restartPolicy: OnFailure
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
--- a/manifests/es-statefulset.yaml
+++ b/manifests/es-statefulset.yaml
@ -0,0 +1,268 @@
+# RBAC authn and authz
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: elasticsearch-logging
+  namespace: kube-system
+  labels:
+    k8s-app: elasticsearch-logging
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: elasticsearch-logging
+  labels:
+    k8s-app: elasticsearch-logging
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "services"
+  - "namespaces"
+  - "endpoints"
+  verbs:
+  - "get"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: kube-system
+  name: elasticsearch-logging
+  labels:
+    k8s-app: elasticsearch-logging
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+- kind: ServiceAccount
+  name: elasticsearch-logging
+  namespace: kube-system
+  apiGroup: ""
+roleRef:
+  kind: ClusterRole
+  name: elasticsearch-logging
+  apiGroup: ""
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: es-config
+  namespace: kube-system
+data:
+  elasticsearch.yml: |
+      path.data: /data/es/data
+      path.logs: /data/es/logs
+      #http.host: 0.0.0.0
+      http.cors.enabled: true
+      http.cors.allow-origin: "*"
+      network.host: 0.0.0.0
+      node.name: ${HOSTNAME}.elasticsearch
+      cluster.initial_master_nodes: ["elasticsearch-0.elasticsearch","elasticsearch-1.elasticsearch"]
+      cluster.name: elasticsearch
+      discovery.zen.ping.unicast.hosts: ["elasticsearch-0.elasticsearch","elasticsearch-1.elasticsearch"]
+      discovery.zen.minimum_master_nodes: 2
+      discovery.zen.ping_timeout: 5s
+
+---
+---
+# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.10.2/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: elasticsearch
+  namespace: kube-system
+  labels:
+    k8s-app: elasticsearch
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  serviceName: elasticsearch
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: elasticsearch
+  template:
+    metadata:
+      labels:
+        k8s-app: elasticsearch
+        kubernetes.io/cluster-service: "true"
+    spec: 
+#      hostNetwork: true
+      tolerations:
+      - operator: "Exists"
+        effect: "NoSchedule"
+        key: 
+      nodeSelector:
+        "node.kubernetes.io/elasticsearch": "true"
+      affinity:
+#        nodeAffinity:
+#        requiredDuringSchedulingIgnoredDuringExecution:
+#         nodeSelectorTerms:
+#          - matchExpressions:
+#            - key: node-role.kubernetes.io/es
+#              operator: Exists
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution: 
+          - labelSelector:
+              matchExpressions:
+              - key: k8s-app
+                operator: In
+                values: 
+                - elasticsearch
+            topologyKey: kubernetes.io/hostname
+      containers:
+      - image: "dev-docker-registry.ccyunchina.com/toyangdon/elasticsearch-oss:7.10.2"
+        name: elasticsearch
+        resources:
+          # need more cpu upon initialization, therefore burstable class
+          limits:
+            cpu: 4
+            memory: "6Gi"
+          requests:
+            cpu: 2
+            memory: 3Gi
+        ports:
+        - containerPort: 9200
+          name: db
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        livenessProbe:
+          tcpSocket:
+            port: transport
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        readinessProbe:
+          tcpSocket:
+            port: transport
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        volumeMounts:
+        - name: es-config
+          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+          subPath: elasticsearch.yml
+        - name: es-persistent-storage
+          mountPath: /data/es
+        env:
+        - name: "NAMESPACE"
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: "ES_JAVA_OPTS"
+          value: "-Xms3g -Xmx3g"
+        securityContext:
+            privileged: true
+      volumes:
+      - name: es-config
+        configMap:
+          name: es-config
+          items:
+          - key: elasticsearch.yml
+            path: elasticsearch.yml
+      initContainers:
+      - image: dev-docker-registry.ccyunchina.com/library/alpine:3.6
+        command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
+        name: elasticsearch-init
+        securityContext:
+          privileged: true
+  volumeClaimTemplates:
+  - metadata:
+      name: es-persistent-storage
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 100Gi
+      storageClassName: local-storage
+      selector:
+        matchLabels:
+          pv: elasticsearch
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    elastic-app: elasticsearch-service
+  name: elasticsearch-service
+  namespace: kube-system
+spec:
+  ports:
+    - port: 9200
+      name: db
+      targetPort: 9200
+    - port: 9300
+      name: transport
+      targetPort: 9300
+  selector:
+    k8s-app: elasticsearch
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    elastic-app: elasticsearch-service
+  name: elasticsearch
+  namespace: kube-system
+spec:
+  ports:
+    - port: 9200
+      name: db
+      targetPort: 9200
+    - port: 9300
+      name: transport
+      targetPort: 9300
+  selector:
+    k8s-app: elasticsearch
+  clusterIP: None
+
+---
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: es-persistent-storage-node-99
+  labels:
+    pv: elasticsearch
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 100Gi
+  local:
+    path: /data/es
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - "node-99"
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: local-storage
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: es-persistent-storage-node-233
+  labels:
+    pv: elasticsearch
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 100Gi
+  local:
+    path: /data/es
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - "node-233"
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: local-storage
--- a/manifests/fluentd.yaml
+++ b/manifests/fluentd.yaml
@ -0,0 +1,481 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd-es
+  labels:
+    k8s-app: fluentd-es
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: fluentd-es
+  labels:
+    k8s-app: fluentd-es
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "namespaces"
+  - "pods"
+  verbs:
+  - "get"
+  - "watch"
+  - "list"
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: fluentd-es
+  labels:
+    k8s-app: fluentd-es
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+- kind: ServiceAccount
+  name: fluentd-es
+  namespace: kube-system
+  apiGroup: ""
+roleRef:
+  kind: ClusterRole
+  name: fluentd-es
+  apiGroup: ""
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd-es-v3.1.1
+  labels:
+    k8s-app: fluentd-es
+    version: v3.1.1
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluentd-es
+      version: v3.1.1
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-es
+        version: v3.1.1
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: fluentd-es
+      containers:
+      - name: fluentd-es
+        image: dev-docker-registry.ccyunchina.com/toyangdon/fluentd-es-image-arm:1.12.0
+        env:
+        - name: FLUENTD_ARGS
+          value: --no-supervisor -q
+        resources:
+          limits:
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: datalog
+          mountPath: /data/log
+        - name: varlibdockercontainers
+          mountPath: /data/docker/containers
+          readOnly: true
+        - name: config-volume
+          mountPath: /etc/fluent/config.d
+        ports:
+        - containerPort: 24231
+          name: prometheus
+          protocol: TCP
+        livenessProbe:
+          tcpSocket:
+            port: prometheus
+          initialDelaySeconds: 5
+          timeoutSeconds: 10
+        readinessProbe:
+          tcpSocket:
+            port: prometheus
+          initialDelaySeconds: 5
+          timeoutSeconds: 10
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: datalog
+        hostPath:
+          path: /data/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /data/docker/containers
+      - name: config-volume
+        configMap:
+          name: fluentd-es-config-v0.2.1
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: fluentd-es-config-v0.2.1
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+data:
+  system.conf: |-
+    <system>
+      root_dir /tmp/fluentd-buffers/
+    </system>
+  containers.input.conf: |-
+    # This configuration file for Fluentd / td-agent is used
+    # to watch changes to Docker log files. The kubelet creates symlinks that
+    # capture the pod name, namespace, container name & Docker container ID
+    # to the docker logs for pods in the /var/log/containers directory on the host.
+    # If running this fluentd configuration in a Docker container, the /var/log
+    # directory should be mounted in the container.
+    #
+    # These logs are then submitted to Elasticsearch which assumes the
+    # installation of the fluent-plugin-elasticsearch & the
+    # fluent-plugin-kubernetes_metadata_filter plugins.
+    # See https://github.com/uken/fluent-plugin-elasticsearch &
+    # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for
+    # more information about the plugins.
+    #
+    # Example
+    # =======
+    # A line in the Docker log file might look like this JSON:
+    #
+    # {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
+    #  "stream":"stderr",
+    #   "time":"2014-09-25T21:15:03.499185026Z"}
+    #
+    # The time_format specification below makes sure we properly
+    # parse the time format produced by Docker. This will be
+    # submitted to Elasticsearch and should appear like:
+    # $ curl 'http://elasticsearch-logging:9200/_search?pretty'
+    # ...
+    # {
+    #      "_index" : "logstash-2014.09.25",
+    #      "_type" : "fluentd",
+    #      "_id" : "VBrbor2QTuGpsQyTCdfzqA",
+    #      "_score" : 1.0,
+    #      "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n",
+    #                 "stream":"stderr","tag":"docker.container.all",
+    #                 "@timestamp":"2014-09-25T22:45:50+00:00"}
+    #    },
+    # ...
+    #
+    # The Kubernetes fluentd plugin is used to write the Kubernetes metadata to the log
+    # record & add labels to the log record if properly configured. This enables users
+    # to filter & search logs on any metadata.
+    # For example a Docker container's logs might be in the directory:
+    #
+    #  /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
+    #
+    # and in the file:
+    #
+    #  997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
+    #
+    # where 997599971ee6... is the Docker ID of the running container.
+    # The Kubernetes kubelet makes a symbolic link to this file on the host machine
+    # in the /var/log/containers directory which includes the pod name and the Kubernetes
+    # container name:
+    #
+    #    synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
+    #    ->
+    #    /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
+    #
+    # The /var/log directory on the host is mapped to the /var/log directory in the container
+    # running this instance of Fluentd and we end up collecting the file:
+    #
+    #   /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
+    #
+    # This results in the tag:
+    #
+    #  var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
+    #
+    # The Kubernetes fluentd plugin is used to extract the namespace, pod name & container name
+    # which are added to the log message as a kubernetes field object & the Docker container ID
+    # is also added under the docker field object.
+    # The final tag is:
+    #
+    #   kubernetes.var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
+    #
+    # And the final log record look like:
+    #
+    # {
+    #   "log":"2014/09/25 21:15:03 Got request with path wombat\n",
+    #   "stream":"stderr",
+    #   "time":"2014-09-25T21:15:03.499185026Z",
+    #   "kubernetes": {
+    #     "namespace": "default",
+    #     "pod_name": "synthetic-logger-0.25lps-pod",
+    #     "container_name": "synth-lgr"
+    #   },
+    #   "docker": {
+    #     "container_id": "997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b"
+    #   }
+    # }
+    #
+    # This makes it easier for users to search for logs by pod name or by
+    # the name of the Kubernetes container regardless of how many times the
+    # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
+    # Json Log Example:
+    # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
+    # CRI Log Example:
+    # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
+    <source>
+      @id fluentd-containers.log
+      @type tail
+      path /var/log/containers/*.log
+      pos_file /var/log/es-containers.log.pos
+      tag raw.kubernetes.*
+      read_from_head true
+      <parse>
+        @type multi_format
+        <pattern>
+          format json
+          time_key time
+          time_format %Y-%m-%dT%H:%M:%S.%NZ
+        </pattern>
+        <pattern>
+          format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
+          time_format %Y-%m-%dT%H:%M:%S.%N%:z
+        </pattern>
+      </parse>
+    </source>
+    # Detect exceptions in the log output and forward them as one log entry.
+    <match raw.kubernetes.**>
+      @id raw.kubernetes
+      @type detect_exceptions
+      remove_tag_prefix raw
+      message log
+      stream stream
+      multiline_flush_interval 5
+      max_bytes 500000
+      max_lines 1000
+    </match>
+    # Concatenate multi-line logs
+    <filter **>
+      @id filter_concat
+      @type concat
+      key message
+      multiline_end_regexp /\n$/
+      separator ""
+    </filter>
+    # Enriches records with Kubernetes metadata
+    <filter kubernetes.**>
+      @id filter_kubernetes_metadata
+      @type kubernetes_metadata
+    </filter>
+    # Fixes json fields in Elasticsearch
+    <filter kubernetes.**>
+      @id filter_parser
+      @type parser
+      key_name log
+      reserve_data true
+      remove_key_name_field true
+      <parse>
+        @type multi_format
+        <pattern>
+          format json
+        </pattern>
+        <pattern>
+          format none
+        </pattern>
+      </parse>
+    </filter>
+  system.input.conf: |-
+    # Example:
+    # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
+    <source>
+      @id minion
+      @type tail
+      format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
+      time_format %Y-%m-%d %H:%M:%S
+      path /var/log/salt/minion
+      pos_file /var/log/salt.pos
+      tag salt
+    </source>
+    # Example:
+    # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
+    <source>
+      @id startupscript.log
+      @type tail
+      format syslog
+      path /var/log/startupscript.log
+      pos_file /var/log/es-startupscript.log.pos
+      tag startupscript
+    </source>
+    # Examples:
+    # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
+    # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
+    # TODO(random-liu): Remove this after cri container runtime rolls out.
+    <source>
+      @id docker.log
+      @type tail
+      format /^time="(?<time>[^"]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
+      path /var/log/docker.log
+      pos_file /var/log/es-docker.log.pos
+      tag docker
+    </source>
+    # Example:
+    # 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
+    <source>
+      @id etcd.log
+      @type tail
+      # Not parsing this, because it doesn't have anything particularly useful to
+      # parse out of it (like severities).
+      format none
+      path /data/log/etcd/etcd.log
+      pos_file /data/log/etcd/es-etcd.log.pos
+      tag etcd
+    </source>
+    # Multi-line parsing is required for all the kube logs because very large log
+    # statements, such as those that include entire object bodies, get split into
+    # multiple lines by glog.
+    # Example:
+    # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
+    <source>
+      @id kubelet.log
+      @type tail
+      format multiline
+      multiline_flush_interval 5s
+      format_firstline /^\w\d{4}/
+      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+      time_format %m%d %H:%M:%S.%N
+      path /data/log/kubelet/kubelet.log
+      pos_file /data/log/kubelet/es-kubelet.log.pos
+      tag kubelet
+    </source>
+    # Example:
+    # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
+    <source>
+      @id glbc.log
+      @type tail
+      format multiline
+      multiline_flush_interval 5s
+      format_firstline /^\w\d{4}/
+      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+      time_format %m%d %H:%M:%S.%N
+      path /var/log/glbc.log
+      pos_file /var/log/es-glbc.log.pos
+      tag glbc
+    </source>
+    # Example:
+    # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+    <source>
+      @id journald-docker
+      @type systemd
+      matches [{ "_SYSTEMD_UNIT": "docker.service" }]
+      <storage>
+        @type local
+        persistent true
+        path /var/log/journald-docker.pos
+      </storage>
+      read_from_head true
+      tag docker
+    </source>
+    <source>
+      @id journald-container-runtime
+      @type systemd
+      matches [{ "_SYSTEMD_UNIT": "docker.service" }]
+      <storage>
+        @type local
+        persistent true
+        path /var/log/journald-container-runtime.pos
+      </storage>
+      read_from_head true
+      tag container-runtime
+    </source>
+    <source>
+      @id journald-kubelet
+      @type systemd
+      matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+      <storage>
+        @type local
+        persistent true
+        path /var/log/journald-kubelet.pos
+      </storage>
+      read_from_head true
+      tag kubelet
+    </source>
+    <source>
+      @id kernel
+      @type systemd
+      matches [{ "_TRANSPORT": "kernel" }]
+      <storage>
+        @type local
+        persistent true
+        path /var/log/kernel.pos
+      </storage>
+      <entry>
+        fields_strip_underscores true
+        fields_lowercase true
+      </entry>
+      read_from_head true
+      tag kernel
+    </source>
+  forward.input.conf: |-
+    # Takes the messages sent over TCP
+    <source>
+      @id forward
+      @type forward
+    </source>
+  monitoring.conf: |-
+    # Prometheus Exporter Plugin
+    # input plugin that exports metrics
+    <source>
+      @id prometheus
+      @type prometheus
+    </source>
+    <source>
+      @id monitor_agent
+      @type monitor_agent
+    </source>
+    # input plugin that collects metrics from MonitorAgent
+    <source>
+      @id prometheus_monitor
+      @type prometheus_monitor
+      <labels>
+        host ${hostname}
+      </labels>
+    </source>
+    # input plugin that collects metrics for output plugin
+    <source>
+      @id prometheus_output_monitor
+      @type prometheus_output_monitor
+      <labels>
+        host ${hostname}
+      </labels>
+    </source>
+    # input plugin that collects metrics for in_tail plugin
+    <source>
+      @id prometheus_tail_monitor
+      @type prometheus_tail_monitor
+      <labels>
+        host ${hostname}
+      </labels>
+    </source>
+  output.conf: |-
+    <match **>
+      @id elasticsearch
+      @type elasticsearch
+      @log_level info
+      type_name _doc
+      include_tag_key true
+      host elasticsearch-service
+      port 9200
+      logstash_format true
+      <buffer>
+        @type file
+        path /var/log/fluentd-buffers/kubernetes.system.buffer
+        flush_mode interval
+        retry_type exponential_backoff
+        flush_thread_count 2
+        flush_interval 5s
+        retry_forever
+        retry_max_interval 30
+        chunk_limit_size 2M
+        total_limit_size 500M
+        overflow_action block
+      </buffer>
+    </match>
--- a/manifests/glusterfs/gluster-storageclass.yaml
+++ b/manifests/glusterfs/gluster-storageclass.yaml
@ -0,0 +1,13 @@
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: gluster-heketi
+  namespace: kube-system
+  annotations: 
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://10.2.1.99:26060"
+  restauthenabled: "false"
+  volumetype: replicate:3
--- a/manifests/glusterfs/glusterfs-daemonset.yaml
+++ b/manifests/glusterfs/glusterfs-daemonset.yaml
@ -0,0 +1,157 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  namespace: kube-system
+  name: glusterfs
+  labels:
+    glusterfs: daemonset
+  annotations:
+    description: GlusterFS DaemonSet
+    tags: glusterfs
+spec:
+  selector:
+    matchLabels:
+      glusterfs: pod
+      glusterfs-node: pod
+  template:
+    metadata:
+      name: glusterfs
+      labels:
+        glusterfs: pod
+        glusterfs-node: pod
+    spec:
+      nodeSelector:
+        "node.kubernetes.io/storage-node": "true"
+      hostNetwork: true
+      tolerations:
+      - key: "node-role.kubernetes.io/storage-node"
+        operator: "Exists"
+        effect: "NoSchedule"
+      hostAliases:
+      - ip: "10.2.1.99"
+        hostnames:
+          - "node-99"
+      - ip: "10.2.1.233"
+        hostnames:
+          - "node-233"
+      - ip: "10.2.1.235"
+        hostnames:
+          - "node-235"
+      hostNetwork: true
+      containers:
+      - image: dev-docker-registry.ccyunchina.com/toyangdon/glusterfs-centos:4-arm64
+        imagePullPolicy: IfNotPresent
+        name: glusterfs
+        env:
+        # alternative for /dev volumeMount to enable access to *all* devices
+        - name: HOST_DEV_DIR
+          value: "/mnt/host-dev"
+        # set GLUSTER_BLOCKD_STATUS_PROBE_ENABLE to "1" so the
+        # readiness/liveness probe validate gluster-blockd as well
+        - name: GLUSTER_BLOCKD_STATUS_PROBE_ENABLE
+          value: "1"
+        - name: GB_GLFS_LRU_COUNT
+          value: "15"
+        - name: TCMU_LOGDIR
+          value: "/var/log/glusterfs/gluster-block"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits:
+            cpu: 2
+            memory: 2Gi
+        volumeMounts:
+        - name: glusterfs-heketi
+          mountPath: "/var/lib/heketi"
+        - name: glusterfs-run
+          mountPath: "/run"
+        - name: glusterfs-lvm
+          mountPath: "/run/lvm"
+        - name: glusterfs-etc
+          mountPath: "/etc/glusterfs"
+        - name: glusterfs-logs
+          mountPath: "/var/log/glusterfs"
+        - name: glusterfs-config
+          mountPath: "/var/lib/glusterd"
+        - name: glusterfs-host-dev
+          mountPath: "/mnt/host-dev"
+        - name: glusterfs-misc
+          mountPath: "/var/lib/misc/glusterfsd"
+        - name: glusterfs-block-sys-class
+          mountPath: "/sys/class"
+        - name: glusterfs-block-sys-module
+          mountPath: "/sys/module"
+        - name: glusterfs-cgroup
+          mountPath: "/sys/fs/cgroup"
+          readOnly: true
+        - name: glusterfs-ssl
+          mountPath: "/etc/ssl"
+          readOnly: true
+        - name: kernel-modules
+          mountPath: "/lib/modules"
+          readOnly: true
+        securityContext:
+          capabilities: {}
+          privileged: true
+        readinessProbe:
+          timeoutSeconds: 3
+          initialDelaySeconds: 40
+          exec:
+            command:
+            - "/bin/bash"
+            - "-c"
+            - "if command -v /usr/local/bin/status-probe.sh; then /usr/local/bin/status-probe.sh readiness; else systemctl status glusterd.service; fi"
+          periodSeconds: 25
+          successThreshold: 1
+          failureThreshold: 50
+        livenessProbe:
+          timeoutSeconds: 3
+          initialDelaySeconds: 40
+          exec:
+            command:
+            - "/bin/bash"
+            - "-c"
+            - "if command -v /usr/local/bin/status-probe.sh; then /usr/local/bin/status-probe.sh liveness; else systemctl status glusterd.service; fi"
+          periodSeconds: 25
+          successThreshold: 1
+          failureThreshold: 50
+      volumes:
+      - name: glusterfs-heketi
+        hostPath:
+          path: "/var/lib/heketi"
+      - name: glusterfs-run
+      - name: glusterfs-lvm
+        hostPath:
+          path: "/run/lvm"
+      - name: glusterfs-etc
+        hostPath:
+          path: "/etc/glusterfs"
+      - name: glusterfs-logs
+        hostPath:
+          path: "/var/log/glusterfs"
+      - name: glusterfs-config
+        hostPath:
+          path: "/var/lib/glusterd"
+      - name: glusterfs-host-dev
+        hostPath:
+          path: "/dev"
+      - name: glusterfs-misc
+        hostPath:
+          path: "/var/lib/misc/glusterfsd"
+      - name: glusterfs-block-sys-class
+        hostPath:
+          path: "/sys/class"
+      - name: glusterfs-block-sys-module
+        hostPath:
+          path: "/sys/module"
+      - name: glusterfs-cgroup
+        hostPath:
+          path: "/sys/fs/cgroup"
+      - name: glusterfs-ssl
+        hostPath:
+          path: "/etc/ssl"
+      - name: kernel-modules
+        hostPath:
+          path: "/lib/modules"
--- a/manifests/glusterfs/heketi-desployment.yaml
+++ b/manifests/glusterfs/heketi-desployment.yaml
@ -0,0 +1,132 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: heketi-service-account
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: heketi-role
+  namespace: kube-system
+  labels:
+    k8s-app: heketi-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: heketi-service-account
+  namespace: kube-system
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: heketi
+  namespace: kube-system
+  labels:
+    glusterfs: heketi-service
+    deploy-heketi: support
+  annotations:
+    description: Exposes Heketi Service
+spec:
+  selector:
+    name: heketi
+  ports:
+  - name: heketi
+    port: 8080
+    targetPort: 8080
+    nodePort: 26060
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata: 
+  annotations: 
+    description: "Defines how to deploy Heketi"
+  labels: 
+    glusterfs: heketi-deployment
+  name: heketi
+  namespace: kube-system
+spec: 
+  selector:
+    matchLabels:
+      glusterfs: heketi-pod
+      name: heketi
+  replicas: 1
+  template: 
+    metadata: 
+      labels: 
+        glusterfs: heketi-pod
+        name: heketi
+      name: heketi
+    spec: 
+      hostAliases:
+      - ip: "10.2.1.99"
+        hostnames:
+          - "node-99"
+      - ip: "10.2.1.233"
+        hostnames:
+          - "node-233"
+      - ip: "10.2.1.235"
+        hostnames:
+          - "node-235"
+      containers: 
+        - 
+          env: 
+            - 
+              name: HEKETI_EXECUTOR
+              value: kubernetes
+            - 
+              name: HEKETI_IGNORE_STALE_OPERATIONS
+              value: "true"
+            - 
+              name: HEKETI_FSTAB
+              value: /var/lib/heketi/fstab
+            - 
+              name: HEKETI_SNAPSHOT_LIMIT
+              value: "14"
+            - 
+              name: HEKETI_KUBE_GLUSTER_DAEMONSET
+              value: "y"
+          image: dev-docker-registry.ccyunchina.com/toyangdon/heketi:8-arm64
+          resources:
+            requests:
+              cpu: 10m
+              memory: 100Mi
+            limits:
+              cpu: 100m
+              memory: 250Mi
+          livenessProbe: 
+            httpGet: 
+              path: /hello
+              port: 8080
+            initialDelaySeconds: 30
+            timeoutSeconds: 3
+          name: heketi
+          ports: 
+            - 
+              containerPort: 8080
+          readinessProbe: 
+            httpGet: 
+              path: /hello
+              port: 8080
+            initialDelaySeconds: 3
+            timeoutSeconds: 3
+          volumeMounts: 
+            - 
+              mountPath: /var/lib/heketi
+              name: db
+      serviceAccountName: heketi-service-account
+      tolerations:
+      - key: "node-role.kubernetes.io/storage-node"
+        operator: "Exists"
+        effect: "NoSchedule"
+      volumes: 
+      - hostPath: 
+          path: /data/heketi
+          type: DirectoryOrCreate
+        name: db
+      nodeSelector:
+        "node.kubernetes.io/heketi": "true"
--- a/manifests/glusterfs/topology.json
+++ b/manifests/glusterfs/topology.json
@ -0,0 +1,53 @@
+{
+  "clusters": [
+	{
+	  "nodes": [
+	  		{
+		  "node": {
+			"hostnames": {
+			  "manage": [
+				"node-99"
+			  ],
+			  "storage": [
+				"10.2.1.99"
+			  ]
+			},
+			"zone": 1
+		  },
+		  "devices": [
+			"/dev/sda"
+		  ]
+		},				{
+		  "node": {
+			"hostnames": {
+			  "manage": [
+				"node-233"
+			  ],
+			  "storage": [
+				"10.2.1.233"
+			  ]
+			},
+			"zone": 1
+		  },
+		  "devices": [
+			"/dev/sda"
+		  ]
+		},				{
+		  "node": {
+			"hostnames": {
+			  "manage": [
+				"node-235"
+			  ],
+			  "storage": [
+				"10.2.1.235"
+			  ]
+			},
+			"zone": 1
+		  },
+		  "devices": [
+			"/dev/sda"
+		  ]
+		}			  ]
+	}
+  ]
+}
--- a/manifests/kibana.yaml
+++ b/manifests/kibana.yaml
@ -0,0 +1,67 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana-logging
+  labels:
+    k8s-app: kibana-logging
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kibana-logging
+  template:
+    metadata:
+      labels:
+        k8s-app: kibana-logging
+    spec:
+      containers:
+        - name: kibana-logging
+          image: "dev-docker-registry.ccyunchina.com/toyangdon/kibana-arm:7.10.2"
+          resources:
+            # need more cpu upon initialization, therefore burstable class
+            limits:
+              cpu: 1000m
+            requests:
+              cpu: 100m
+          env:
+            - name: ELASTICSEARCH_HOSTS
+              value: http://elasticsearch-service:9200
+            - name: SERVER_NAME
+              value: kibana-logging
+          ports:
+            - containerPort: 5601
+              name: ui
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /api/status
+              port: ui
+            initialDelaySeconds: 5
+            timeoutSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /api/status
+              port: ui
+            initialDelaySeconds: 5
+            timeoutSeconds: 10
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana-logging
+  labels:
+    k8s-app: kibana-logging
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "Kibana"
+spec:
+  ports:
+  - port: 5601
+    protocol: TCP
+    targetPort: ui
+    nodePort: 25601
+  selector:
+    k8s-app: kibana-logging
+  type: NodePort
--- a/manifests/kube-proxy.yaml
+++ b/manifests/kube-proxy.yaml
@ -0,0 +1,110 @@
+# Please keep kube-proxy configuration in-sync with:
+# cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-proxy
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-proxy
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 10%
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: "Exists"
+        effect: "NoExecute"
+      - operator: "Exists"
+        effect: "NoSchedule"
+      containers:
+      - name: kube-proxy
+        image: dev-docker-registry.ccyunchina.com/toyangdon/kube-proxy-arm64:v1.21.0
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+          limits:
+            cpu: 200m
+            memory: 400Mi
+        livenessProbe:
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 10256
+          failureThreshold: 8
+          initialDelaySeconds: 15
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 15
+        command:
+        - /bin/sh
+        - -c
+        - kube-proxy --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig --v=2  --alsologtostderr --logtostderr=false --log-dir=/data/log/kube-proxy --healthz-bind-address=127.0.0.1
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /data/log/kube-proxy
+          name: varlog
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+          readOnly: false
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+        - mountPath: /etc/kubernetes/kube-proxy.kubeconfig
+          name: kube-config
+          readOnly: true
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /data/log/kube-proxy
+          type: DirectoryOrCreate
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: kube-config
+        hostPath:
+          path: /etc/kubernetes/kube-proxy.kubeconfig
+      serviceAccountName: kube-proxy
+      
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: system:kube-proxy
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+  - kind: ServiceAccount
+    name: kube-proxy
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: system:node-proxier
+  apiGroup: rbac.authorization.k8s.io
--- a/manifests/kubernetes-dashboard.yaml
+++ b/manifests/kubernetes-dashboard.yaml
@ -0,0 +1,320 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: kube-system
+
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+    kubernetes.io/cluster-service: "true"
+  name: kubernetes-dashboard
+  namespace: kube-system
+spec:
+  ports:
+    - port: 443
+      targetPort: 8443
+      nodePort: 16060
+  selector:
+    k8s-app: kubernetes-dashboard
+  type: NodePort
+
+---
+
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-certs
+  namespace: kube-system
+type: Opaque
+
+---
+
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-csrf
+  namespace: kube-system
+type: Opaque
+data:
+  csrf: ""
+
+---
+
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-key-holder
+  namespace: kube-system
+type: Opaque
+
+---
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard-settings
+  namespace: kube-system
+
+---
+
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: kube-system
+rules:
+  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
+    verbs: ["get", "update", "delete"]
+    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["kubernetes-dashboard-settings"]
+    verbs: ["get", "update"]
+    # Allow Dashboard to get metrics.
+  - apiGroups: [""]
+    resources: ["services"]
+    resourceNames: ["heapster", "dashboard-metrics-scraper"]
+    verbs: ["proxy"]
+  - apiGroups: [""]
+    resources: ["services/proxy"]
+    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
+    verbs: ["get"]
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+rules:
+  # Allow Metrics Scraper to get metrics from the Metrics server
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["pods", "nodes"]
+    verbs: ["get", "list", "watch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubernetes-dashboard
+subjects:
+  - kind: ServiceAccount
+    name: kubernetes-dashboard
+    namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-dashboard
+subjects:
+  - kind: ServiceAccount
+    name: kubernetes-dashboard
+    namespace: kube-system
+
+---
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  labels:
+    k8s-app: kubernetes-dashboard
+  name: kubernetes-dashboard
+  namespace: kube-system
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      k8s-app: kubernetes-dashboard
+  template:
+    metadata:
+      labels:
+        k8s-app: kubernetes-dashboard
+    spec:
+      containers:
+        - name: kubernetes-dashboard
+          image: dev-docker-registry.ccyunchina.com/kubernetesui/dashboard:v2.0.0
+          ports:
+            - containerPort: 8443
+              protocol: TCP
+          args:
+            - --auto-generate-certificates
+            - --namespace=kube-system
+            # Uncomment the following line to manually specify Kubernetes API server Host
+            # If not specified, Dashboard will attempt to auto discover the API server and connect
+            # to it. Uncomment only if the default does not work.
+            # - --apiserver-host=http://my-address:port
+          volumeMounts:
+            - name: kubernetes-dashboard-certs
+              mountPath: /certs
+              # Create on-disk volume to store exec logs
+            - mountPath: /tmp
+              name: tmp-volume
+          livenessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: /
+              port: 8443
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsUser: 1001
+            runAsGroup: 2001
+      volumes:
+        - name: kubernetes-dashboard-certs
+          secret:
+            secretName: kubernetes-dashboard-certs
+        - name: tmp-volume
+          emptyDir: {}
+      serviceAccountName: kubernetes-dashboard
+      nodeSelector:
+        "beta.kubernetes.io/os": linux
+      # Comment the following tolerations if Dashboard must not be deployed on master
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    k8s-app: dashboard-metrics-scraper
+  name: dashboard-metrics-scraper
+  namespace: kube-system
+spec:
+  ports:
+    - port: 8000
+      targetPort: 8000
+  selector:
+    k8s-app: dashboard-metrics-scraper
+
+---
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  labels:
+    k8s-app: dashboard-metrics-scraper
+  name: dashboard-metrics-scraper
+  namespace: kube-system
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      k8s-app: dashboard-metrics-scraper
+  template:
+    metadata:
+      labels:
+        k8s-app: dashboard-metrics-scraper
+      annotations:
+        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
+    spec:
+      containers:
+        - name: dashboard-metrics-scraper
+          image: dev-docker-registry.ccyunchina.com/kubernetesui/metrics-scraper:v1.0.2
+          ports:
+            - containerPort: 8000
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              scheme: HTTP
+              path: /
+              port: 8000
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          volumeMounts:
+          - mountPath: /tmp
+            name: tmp-volume
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsUser: 1001
+            runAsGroup: 2001
+      serviceAccountName: kubernetes-dashboard
+      nodeSelector:
+        "beta.kubernetes.io/os": linux
+      # Comment the following tolerations if Dashboard must not be deployed on master
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      volumes:
+        - name: tmp-volume
+          emptyDir: {}
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-user
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: admin-user
+  namespace: kube-system
--- a/manifests/local-storage.yaml
+++ b/manifests/local-storage.yaml
@ -0,0 +1,6 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: local-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
--- a/manifests/metrics-server.yaml
+++ b/manifests/metrics-server.yaml
@ -0,0 +1,143 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+
+---
+
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
+      containers:
+      - name: metrics-server
+        #image: k8s.gcr.io/metrics-server-amd64:v0.3.6
+        image: dev-docker-registry.ccyunchina.com/toyangdon/metrics-server-arm64:v0.3.4
+        imagePullPolicy: IfNotPresent 
+        command:
+        - /metrics-server
+        - --metric-resolution=30s
+        - --kubelet-insecure-tls
+        - '--kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP'
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: 443
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "extensions"
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
--- a/manifests/openvpn.yaml
+++ b/manifests/openvpn.yaml
@ -0,0 +1,256 @@
+---
+apiVersion: v1
+data:
+  VPN.SERVERNAME.COM.crt: "Certificate:\n    Data:\n        Version: 3 (0x2)\n        Serial
+    Number: 1 (0x1)\n    Signature Algorithm: sha256WithRSAEncryption\n        Issuer:
+    CN=Easy-RSA CA\n        Validity\n            Not Before: Sep 18 08:32:08 2017
+    GMT\n            Not After : Sep 16 08:32:08 2027 GMT\n        Subject: CN=VPN.SERVERNAME.COM\n
+    \       Subject Public Key Info:\n            Public Key Algorithm: rsaEncryption\n
+    \               Public-Key: (2048 bit)\n                Modulus:\n                    00:ce:b7:63:6e:8c:ac:9b:83:37:20:a1:d8:59:6f:\n
+    \                   62:03:4a:fb:9d:56:c9:9c:38:f2:b5:33:30:20:ff:\n                    82:7f:bf:69:af:a6:99:74:cc:28:55:bf:44:95:a1:\n
+    \                   b7:6f:08:25:01:8c:53:22:62:00:c2:8f:4f:95:89:\n                    00:ae:6b:07:d6:eb:83:b7:7f:82:0b:0f:82:f2:ad:\n
+    \                   71:e3:7c:3a:f8:b1:39:24:ad:c7:f6:e3:b3:f2:21:\n                    54:90:91:f0:07:26:7a:b2:a0:25:ca:a8:d5:b1:c6:\n
+    \                   eb:34:91:31:24:83:84:c4:b7:9f:f6:91:d1:72:b4:\n                    69:c1:cf:8e:8d:23:d4:01:af:72:75:cc:5b:10:61:\n
+    \                   a6:2b:4f:f6:7a:10:3d:89:2c:a5:92:65:5d:c1:e0:\n                    c5:f0:6b:fc:8e:7b:d0:1c:c1:9a:02:e4:66:1f:21:\n
+    \                   11:b8:e1:26:cb:83:04:58:89:2c:3d:04:8e:05:8c:\n                    32:a2:18:7b:c2:d5:ec:9a:85:42:d0:8e:a2:b4:ff:\n
+    \                   45:55:03:50:c4:e1:1e:c9:77:6d:80:56:10:52:17:\n                    d5:c8:10:8a:93:50:4c:a1:49:74:55:97:2e:35:cc:\n
+    \                   51:fd:f7:81:a0:db:9c:a0:fd:5b:7b:3f:b6:4f:1c:\n                    11:73:e8:d3:bb:cc:fe:ea:41:ea:2f:98:1a:e1:79:\n
+    \                   99:3f\n                Exponent: 65537 (0x10001)\n        X509v3
+    extensions:\n            X509v3 Basic Constraints: \n                CA:FALSE\n
+    \           X509v3 Subject Key Identifier: \n                D7:84:50:0E:4C:AC:4D:31:82:79:9E:B4:F1:02:9F:08:B5:6E:BA:4A\n
+    \           X509v3 Authority Key Identifier: \n                keyid:F3:77:AD:D7:27:1E:47:D2:C9:4B:5D:F0:D3:9F:90:76:18:3A:84:70\n
+    \               DirName:/CN=Easy-RSA CA\n                serial:B8:8F:08:51:CB:16:13:4F\n\n
+    \           X509v3 Extended Key Usage: \n                TLS Web Server Authentication\n
+    \           X509v3 Key Usage: \n                Digital Signature, Key Encipherment\n
+    \   Signature Algorithm: sha256WithRSAEncryption\n         a3:ff:a5:19:9f:e2:69:ba:2c:89:6e:15:4b:e3:aa:c0:3d:79:\n
+    \        28:e4:dd:6a:e0:9a:8e:73:c6:aa:8a:91:b8:b4:26:1b:08:8a:\n         78:26:10:5e:e6:8d:20:8c:01:2c:f1:69:d3:80:51:d3:79:68:\n
+    \        15:aa:9f:5c:c0:8b:92:03:88:f8:9c:ce:05:ea:cd:fb:74:33:\n         36:f7:b6:07:a0:bf:57:43:61:6e:42:f3:3a:a4:e4:b9:d8:e1:\n
+    \        9c:02:77:0a:4e:83:ed:a3:73:3b:c9:78:f8:79:5d:29:4d:c5:\n         08:5e:e6:5a:64:f7:de:19:75:a1:3e:aa:47:ae:b8:9c:70:cf:\n
+    \        d5:88:52:ff:70:b5:c7:89:03:19:ce:0b:1b:ee:f2:49:e1:21:\n         36:52:d2:e1:21:8f:a6:52:fd:62:54:a4:97:80:45:90:ce:06:\n
+    \        70:d7:34:43:d0:ff:7a:37:03:f3:08:b7:df:08:6a:00:ee:51:\n         3e:3c:52:b2:f0:f7:1c:80:cb:ff:f3:fd:8e:1e:a3:aa:9d:30:\n
+    \        12:7b:55:09:81:7b:66:db:99:29:a1:7e:2e:13:5c:7d:db:a6:\n         0a:34:a0:66:f2:9c:b8:86:af:11:9e:1a:46:ab:c5:60:b4:85:\n
+    \        9c:db:7e:8b:98:32:69:cf:25:2d:fe:9f:e6:ca:5e:42:4d:1f:\n         56:a8:9c:43\n-----BEGIN
+    CERTIFICATE-----\nMIIDRjCCAi6gAwIBAgIBATANBgkqhkiG9w0BAQsFADAWMRQwEgYDVQQDDAtFYXN5\nLVJTQSBDQTAeFw0xNzA5MTgwODMyMDhaFw0yNzA5MTYwODMyMDhaMB0xGzAZBgNV\nBAMMElZQTi5TRVJWRVJOQU1FLkNPTTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAM63Y26MrJuDNyCh2FlvYgNK+51WyZw48rUzMCD/gn+/aa+mmXTMKFW/\nRJWht28IJQGMUyJiAMKPT5WJAK5rB9brg7d/ggsPgvKtceN8OvixOSStx/bjs/Ih\nVJCR8AcmerKgJcqo1bHG6zSRMSSDhMS3n/aR0XK0acHPjo0j1AGvcnXMWxBhpitP\n9noQPYkspZJlXcHgxfBr/I570BzBmgLkZh8hEbjhJsuDBFiJLD0EjgWMMqIYe8LV\n7JqFQtCOorT/RVUDUMThHsl3bYBWEFIX1cgQipNQTKFJdFWXLjXMUf33gaDbnKD9\nW3s/tk8cEXPo07vM/upB6i+YGuF5mT8CAwEAAaOBlzCBlDAJBgNVHRMEAjAAMB0G\nA1UdDgQWBBTXhFAOTKxNMYJ5nrTxAp8ItW66SjBGBgNVHSMEPzA9gBTzd63XJx5H\n0slLXfDTn5B2GDqEcKEapBgwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0GCCQC4jwhR\nyxYTTzATBgNVHSUEDDAKBggrBgEFBQcDATALBgNVHQ8EBAMCBaAwDQYJKoZIhvcN\nAQELBQADggEBAKP/pRmf4mm6LIluFUvjqsA9eSjk3Wrgmo5zxqqKkbi0JhsIingm\nEF7mjSCMASzxadOAUdN5aBWqn1zAi5IDiPiczgXqzft0Mzb3tgegv1dDYW5C8zqk\n5LnY4ZwCdwpOg+2jczvJePh5XSlNxQhe5lpk994ZdaE+qkeuuJxwz9WIUv9wtceJ\nAxnOCxvu8knhITZS0uEhj6ZS/WJUpJeARZDOBnDXNEPQ/3o3A/MIt98IagDuUT48\nUrLw9xyAy//z/Y4eo6qdMBJ7VQmBe2bbmSmhfi4TXH3bpgo0oGbynLiGrxGeGkar\nxWC0hZzbfouYMmnPJS3+n+bKXkJNH1aonEM=\n-----END
+    CERTIFICATE-----\n"
+  VPN.SERVERNAME.COM.key: |
+    -----BEGIN PRIVATE KEY-----
+    MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDOt2NujKybgzcg
+    odhZb2IDSvudVsmcOPK1MzAg/4J/v2mvppl0zChVv0SVobdvCCUBjFMiYgDCj0+V
+    iQCuawfW64O3f4ILD4LyrXHjfDr4sTkkrcf247PyIVSQkfAHJnqyoCXKqNWxxus0
+    kTEkg4TEt5/2kdFytGnBz46NI9QBr3J1zFsQYaYrT/Z6ED2JLKWSZV3B4MXwa/yO
+    e9AcwZoC5GYfIRG44SbLgwRYiSw9BI4FjDKiGHvC1eyahULQjqK0/0VVA1DE4R7J
+    d22AVhBSF9XIEIqTUEyhSXRVly41zFH994Gg25yg/Vt7P7ZPHBFz6NO7zP7qQeov
+    mBrheZk/AgMBAAECggEANKpiadLrUv+eNvElBrwq833SFyb6XJPhuiqdc3rFi7h/
+    u6GtbSVqjVYaqii2bFJd40QWFidz3B3Y/9cU1lssdyQvvbt9rUW5bgykfHm1KJE2
+    ZtLjpFIlEWZOfjrANYInNoU4u6ZCM3XjjNQE+jGLzPafiZljDF9MpnUz81+Sbw8H
+    XuPYpxF+NzHaRxe5TO54Scbn/a78LqKWBvBfexZF+eW8yq6cpQELfxV86osG+1cQ
+    Ln+lu08dUWCBYD+oYK1WJBmN0L6c41URLhVQJlPcJw59bJKblS5WBNstgwXs/nBd
+    1F/ExmZHmH7vHxUq9302m4YMLSibSl1qZ/nL4btpcQKBgQD7yzS1sRF2682EKie1
+    Zzxy+SGUsNwS64OlcYZrZjlSv9PlNkC5sbXgdie9uNTHJDjH9n+/ZBBtaSQIOEHZ
+    /LXriyB5K+4h5YbO7FTP5A3mkjbE5NDho0H2HJbKiBfnQikm5OIv7g0QypkhzemS
+    PH15YWntj4ch70f7X/ttnsnzpwKBgQDSK2jIZGLLGocJ9nX6ypBuFVwbDbEQlNlN
+    /3AqmOoci0xhUbIY5XVh5gfK6J/+w8tOQrCTo8Mk3Z1tdOA/SamqAH5gEIt50Qxy
+    no6wIxy4SxWxlRm51kDaqakbErrvH7YtEIDjB2C49pibCSAURUYkReD7rkMBKiCD
+    DQ9ZBJJAqQKBgQCcfSeuQHVFPrpFCNoTAS1tm26GdgDg9WSlP1A2kZrQ1g4Ap95E
+    SrFp5qU+YZBEkDuF86pjrfITvcO6JBSxQUFjOd2d0XK3uGo3UmRHrEoDYh4N8tPr
+    WEbXid8h0sFgZhsjZB87gKy4IeJYQxyGWXf2J1fpYEmxCoDnNzkRYR3MgwKBgQC5
+    k3L2Xw2M4WOTvHf4dpyJV3r4uGI8jfJDGOXl9xfhFG7hD83+hb3cL815sWZ1AlWm
+    SOS5uMQ67SRw3d6Ug2q4na75DCLb8WbYXoEHrjlR0ZraLxmEKlchvOWA7kz5kDrA
+    F+52iLEzISsm2ivlOjXxGXLLSNgJUzVaAlIm/A5X4QKBgQCi6Lv7I9+FZK9KzWBQ
+    1oj3xhjOA/ibt4tu7Vu+JpaDq0Y12Gltgi89llBlc5MISvPxy9c72u8SCmyPe71d
+    kt2UgvUoG3RfDjbVS+BExW0UQ7rhOTHfHZMqqSBh8oSmBB9MIJ+VB9bJyZwZOtto
+    TTHWyNxB+mrHGTcJHHAv5MX6yw==
+    -----END PRIVATE KEY-----
+  ca.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIDNTCCAh2gAwIBAgIJALiPCFHLFhNPMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV
+    BAMMC0Vhc3ktUlNBIENBMB4XDTE3MDkxODA4MzEwN1oXDTI3MDkxNjA4MzEwN1ow
+    FjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+    ggEKAoIBAQC+sLzhdSrVCiU+8lkPgKd9MC6Q7zlgislvb+uEAwb4fspUSraPjCRl
+    gcAfSmHur+/MTPQK2pB58wMkm3g6vRkJt0E0wY76trZKVa8YjnSweEde+dxxeLXD
+    iIpoJFBhox/lgy3ddx2hEQHrlv7P4NZ9hRWpLZo3WMTw2OMfLrDPRyhQfBJosy0W
+    goJfdeB9UYLsZgfejPpILTaUy//ukFgoQMrSH/O98ZikbA8SdviRqjp03tqjzxV6
+    0Elcf2QLNbC1wiZTngdwtAVOJKgaGw1bBTrLWbck8Yk7bXatkEi86/l7um2iHOPD
+    9Y6kNrwCKTDBrM7+tKbdEPTiOuri2mO1AgMBAAGjgYUwgYIwHQYDVR0OBBYEFPN3
+    rdcnHkfSyUtd8NOfkHYYOoRwMEYGA1UdIwQ/MD2AFPN3rdcnHkfSyUtd8NOfkHYY
+    OoRwoRqkGDAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQYIJALiPCFHLFhNPMAwGA1Ud
+    EwQFMAMBAf8wCwYDVR0PBAQDAgEGMA0GCSqGSIb3DQEBCwUAA4IBAQBgLNkq8wUr
+    cVBy008ojgfm0Ko/QJBRWcL0Yrl/r7N871KuDhLssItZ7aWZIg0j8deAfNrCVgCt
+    tpaGB2HC+yzhuEHRHlPCeoSw4O+xTS5AmJB9Q9WjRFP+yzJ7FIA4op3I8ZxvZK1w
+    nXVuAKkwyIrFM6yz5B/ift/t5Jxt6BaFdYjzOnQw5DbECh/PKhD0C4QUR28zVHKW
+    53xXDuPqiltIf94bJAf8Ct2HnGcQMWyqw1n6Bu0H73+ecG3XJxom8NOuWrGOPPSO
+    HnCoW35mnCgrKZ5YhJevpKDjuG6QbGtEs+caVWZOENFPBRe5tDAvyH2oMiPPe2CJ
+    5WjdLeyPTMZ0
+    -----END CERTIFICATE-----
+  dh.pem: |
+    -----BEGIN DH PARAMETERS-----
+    MIIBCAKCAQEAx7QV8Js07OH36V5pbPNVFsq5kLHz6QMnmN3f3pwQHtAl96wvcFeB
+    NOwMIVPKDiJjiRaWVpL42LryTbEyuWeAbYkgdU0fe6fZDLGLrGAwCEnIXBtAfaKt
+    829h6Tm0APaordl07KkWRDl8SYs/gZcgO0r7tC8kRCzHC5240pKWQg+dl1rZo+xh
+    2Zwu70RijW6pIXwUQ1vPp72muSpwLcelAdubuti3ASTl06aXjcaVvMRXkknXm8GP
+    V25irZq5qxq2EOQn9+V1Zr0qk+gN+cf2BwdS71eTbNhHmDofKxwUeYX7unKO+iPa
+    c5cal9OTocKVxYs03/0soF+qiDIhHVEa+wIBAg==
+    -----END DH PARAMETERS-----
+  openvpn.conf: |
+    server 192.168.255.0 255.255.255.0
+    verb 3
+    key /etc/openvpn/VPN.SERVERNAME.COM.key
+    ca /etc/openvpn/ca.crt
+    cert /etc/openvpn/VPN.SERVERNAME.COM.crt
+    dh /etc/openvpn/dh.pem
+    tls-auth /etc/openvpn/ta.key
+    key-direction 0
+    keepalive 10 60
+    persist-key
+    persist-tun
+
+    proto udp
+    # Rely on Docker to do port mapping, internally always 1194
+    port 1194
+    dev tun0
+    status /tmp/openvpn-status.log
+
+    user nobody
+    group nogroup
+
+    ### Route Configurations Below
+    route 192.168.254.0 255.255.255.0
+
+    ### Push Configurations Below
+    #push "block-outside-dns"
+    #push "dhcp-option DNS 8.8.8.8"
+    #push "dhcp-option DNS 8.8.4.4"
+
+    duplicate-cn
+  ovpn_env.sh: |
+    declare -x OVPN_AUTH=
+    declare -x OVPN_CIPHER=
+    declare -x OVPN_CLIENT_TO_CLIENT=
+    declare -x OVPN_CN=VPN.SERVERNAME.COM
+    declare -x OVPN_COMP_LZO=0
+    declare -x OVPN_DEFROUTE=1
+    declare -x OVPN_DEVICE=tun
+    declare -x OVPN_DEVICEN=0
+    declare -x OVPN_DISABLE_PUSH_BLOCK_DNS=0
+    declare -x OVPN_DNS=1
+    declare -x OVPN_DNS_SERVERS=([0]="8.8.8.8" [1]="8.8.4.4")
+    declare -x OVPN_ENV=/etc/openvpn/ovpn_env.sh
+    declare -x OVPN_EXTRA_CLIENT_CONFIG=()
+    declare -x OVPN_EXTRA_SERVER_CONFIG=()
+    declare -x OVPN_FRAGMENT=
+    declare -x OVPN_KEEPALIVE='10 60'
+    declare -x OVPN_MTU=
+    declare -x OVPN_NAT=0
+    declare -x OVPN_PORT=1194
+    declare -x OVPN_PROTO=udp
+    declare -x OVPN_PUSH=()
+    declare -x OVPN_ROUTES=([0]="192.168.254.0/24")
+    declare -x OVPN_SERVER=192.168.255.0/24
+    declare -x OVPN_SERVER_URL=udp://VPN.SERVERNAME.COM
+    declare -x OVPN_TLS_CIPHER=
+  ta.key: |
+    #
+    # 2048 bit OpenVPN static key
+    #
+    -----BEGIN OpenVPN Static key V1-----
+    39fdc43386183dec5048fce378903cf4
+    5180b71ce22e4994bbca8b2f89d39ded
+    48e7d5b2a7a32db48490d3b0ef873803
+    8ae0d25471dc8f2fbf3c4e3bec3250a3
+    8601a37393b1dc9f39f096aeb15e0e44
+    363a2429576cba594848ed15434f08e0
+    ae656d190a3027b70df7566b3f029c3a
+    9e968d20ad984d7a48c5f2b2d92e870e
+    25ae2ba6293ac3a7fb2c1abac04ff60c
+    e36147a3d072f388e87d128d16b4f09c
+    479b8b93f5ae16b4351d9ffc60eee34f
+    152e5f2fb647c7d630bb4136d196b47e
+    0eae07e282e7acf6f61e13684679a9d0
+    f3601c69d3443b377e21803c9f63062f
+    671c1b13ba3ee8e872f12236a60a826a
+    bbd178f272829eec64258b3199a05134
+    -----END OpenVPN Static key V1-----
+kind: ConfigMap
+metadata:
+  creationTimestamp: 2018-04-03T07:13:27Z
+  name: openvpn-configmap
+  namespace: kube-public
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: openvpn
+  name: openvpn
+  namespace: kube-public
+spec:
+  ports:
+  - name: port1
+    nodePort: 11940
+    port: 1194
+    protocol: UDP
+    targetPort: 1194
+  selector:
+    app: openvpn
+  type: NodePort
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: openvpn
+  name: openvpn
+  namespace: kube-public
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openvpn
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: openvpn
+    spec:
+      initContainers:
+      - command:
+        - /sbin/sysctl
+        - -w
+        - net.ipv4.ip_forward=1
+        image: dev-docker-registry.ccyunchina.com/toyangdon/openvpn-arm:20191120
+        name: init-containers
+        securityContext:
+          privileged: true
+      containers:
+      - image: dev-docker-registry.ccyunchina.com/toyangdon/openvpn-arm:20191120
+        imagePullPolicy: IfNotPresent
+        name: openvpn-container1
+        ports:
+        - containerPort: 1194
+          protocol: UDP
+        resources: {}
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        resources:
+          requests:
+            cpu: 30m
+            memory: 10Mi
+          limits:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - mountPath: /etc/openvpn
+          name: openvpn-configmap
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: openvpn-configmap
+        name: openvpn-configmap
--- a/manifests/prometheus/@
+++ b/manifests/prometheus/@
@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: alertmanager
+  name: alertmanager
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: web
+  selector:
+    matchLabels:
+      alertmanager: main
--- a/manifests/prometheus/alertmanager-alertmanager.yaml
+++ b/manifests/prometheus/alertmanager-alertmanager.yaml
@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  labels:
+    alertmanager: main
+  name: main
+  namespace: monitoring
+spec:
+  baseImage: dev-docker-registry.ccyunchina.com/prom/alertmanager
+  nodeSelector:
+    kubernetes.io/os: linux
+  replicas: 3
+  securityContext:
+    fsGroup: 2000
+    runAsNonRoot: true
+    runAsUser: 1000
+  serviceAccountName: alertmanager-main
+  version: v0.18.0
--- a/manifests/prometheus/alertmanager-pvc.yaml.bak
+++ b/manifests/prometheus/alertmanager-pvc.yaml.bak
@ -0,0 +1,9 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: prometheus-pv
+spec:
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
--- a/manifests/prometheus/alertmanager-secret.yaml
+++ b/manifests/prometheus/alertmanager-secret.yaml
@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  alertmanager.yaml: Imdsb2JhbCI6CiAgInJlc29sdmVfdGltZW91dCI6ICI1bSIKInJlY2VpdmVycyI6Ci0gIm5hbWUiOiAibnVsbCIKInJvdXRlIjoKICAiZ3JvdXBfYnkiOgogIC0gImpvYiIKICAiZ3JvdXBfaW50ZXJ2YWwiOiAiNW0iCiAgImdyb3VwX3dhaXQiOiAiMzBzIgogICJyZWNlaXZlciI6ICJudWxsIgogICJyZXBlYXRfaW50ZXJ2YWwiOiAiMTJoIgogICJyb3V0ZXMiOgogIC0gIm1hdGNoIjoKICAgICAgImFsZXJ0bmFtZSI6ICJXYXRjaGRvZyIKICAgICJyZWNlaXZlciI6ICJudWxsIg==
+kind: Secret
+metadata:
+  name: alertmanager-main
+  namespace: monitoring
+type: Opaque
--- a/manifests/prometheus/alertmanager-service.yaml
+++ b/manifests/prometheus/alertmanager-service.yaml
@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    alertmanager: main
+  name: alertmanager-main
+  namespace: monitoring
+spec:
+  ports:
+  - name: web
+    nodePort: 22093
+    port: 9093
+    targetPort: web
+  selector:
+    alertmanager: main
+    app: alertmanager
+  sessionAffinity: ClientIP
+  type: NodePort
--- a/manifests/prometheus/alertmanager-serviceAccount.yaml
+++ b/manifests/prometheus/alertmanager-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: alertmanager-main
+  namespace: monitoring
--- a/manifests/prometheus/alertmanager-serviceMonitor.yaml
+++ b/manifests/prometheus/alertmanager-serviceMonitor.yaml
@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: alertmanager
+  name: alertmanager
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: web
+  selector:
+    matchLabels:
+      alertmanager: main
--- a/manifests/prometheus/grafana-dashboardDatasources.yaml
+++ b/manifests/prometheus/grafana-dashboardDatasources.yaml
@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  datasources.yaml: ewogICAgImFwaVZlcnNpb24iOiAxLAogICAgImRhdGFzb3VyY2VzIjogWwogICAgICAgIHsKICAgICAgICAgICAgImFjY2VzcyI6ICJwcm94eSIsCiAgICAgICAgICAgICJlZGl0YWJsZSI6IGZhbHNlLAogICAgICAgICAgICAibmFtZSI6ICJwcm9tZXRoZXVzIiwKICAgICAgICAgICAgIm9yZ0lkIjogMSwKICAgICAgICAgICAgInR5cGUiOiAicHJvbWV0aGV1cyIsCiAgICAgICAgICAgICJ1cmwiOiAiaHR0cDovL3Byb21ldGhldXMtazhzLm1vbml0b3Jpbmcuc3ZjOjkwOTAiLAogICAgICAgICAgICAidmVyc2lvbiI6IDEKICAgICAgICB9CiAgICBdCn0=
+kind: Secret
+metadata:
+  name: grafana-datasources
+  namespace: monitoring
+type: Opaque
--- a/manifests/prometheus/grafana-dashboardDefinitions.yaml
+++ b/manifests/prometheus/grafana-dashboardDefinitions.yaml
--- a/manifests/prometheus/grafana-dashboardSources.yaml
+++ b/manifests/prometheus/grafana-dashboardSources.yaml
@ -0,0 +1,21 @@
+apiVersion: v1
+data:
+  dashboards.yaml: |-
+    {
+        "apiVersion": 1,
+        "providers": [
+            {
+                "folder": "",
+                "name": "0",
+                "options": {
+                    "path": "/grafana-dashboard-definitions/0"
+                },
+                "orgId": 1,
+                "type": "file"
+            }
+        ]
+    }
+kind: ConfigMap
+metadata:
+  name: grafana-dashboards
+  namespace: monitoring
--- a/manifests/prometheus/grafana-deployment.yaml
+++ b/manifests/prometheus/grafana-deployment.yaml
@ -0,0 +1,216 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: grafana
+  name: grafana
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      containers:
+      - image: dev-docker-registry.ccyunchina.com/grafana/grafana:6.6.2
+        name: grafana
+        ports:
+        - containerPort: 3000
+          name: http
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: http
+        resources:
+          limits:
+            cpu: 200m
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - mountPath: /var/lib/grafana
+          name: grafana-storage
+          readOnly: false
+        - mountPath: /etc/grafana/provisioning/datasources
+          name: grafana-datasources
+          readOnly: false
+        - mountPath: /etc/grafana/provisioning/dashboards
+          name: grafana-dashboards
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/apiserver
+          name: grafana-dashboard-apiserver
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/cluster-total
+          name: grafana-dashboard-cluster-total
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/controller-manager
+          name: grafana-dashboard-controller-manager
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-cluster
+          name: grafana-dashboard-k8s-resources-cluster
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-namespace
+          name: grafana-dashboard-k8s-resources-namespace
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-node
+          name: grafana-dashboard-k8s-resources-node
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-pod
+          name: grafana-dashboard-k8s-resources-pod
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-workload
+          name: grafana-dashboard-k8s-resources-workload
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/k8s-resources-workloads-namespace
+          name: grafana-dashboard-k8s-resources-workloads-namespace
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/kubelet
+          name: grafana-dashboard-kubelet
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/namespace-by-pod
+          name: grafana-dashboard-namespace-by-pod
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/namespace-by-workload
+          name: grafana-dashboard-namespace-by-workload
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/node-cluster-rsrc-use
+          name: grafana-dashboard-node-cluster-rsrc-use
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/node-rsrc-use
+          name: grafana-dashboard-node-rsrc-use
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/nodes
+          name: grafana-dashboard-nodes
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/persistentvolumesusage
+          name: grafana-dashboard-persistentvolumesusage
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/pod-total
+          name: grafana-dashboard-pod-total
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/pods
+          name: grafana-dashboard-pods
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/prometheus-remote-write
+          name: grafana-dashboard-prometheus-remote-write
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/prometheus
+          name: grafana-dashboard-prometheus
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/proxy
+          name: grafana-dashboard-proxy
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/scheduler
+          name: grafana-dashboard-scheduler
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/statefulset
+          name: grafana-dashboard-statefulset
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/workload-total
+          name: grafana-dashboard-workload-total
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/rocketmq
+          name: grafana-dashboard-rocketmq
+          readOnly: false
+        - mountPath: /grafana-dashboard-definitions/0/mysql
+          name: grafana-dashboard-mysql
+          readOnly: false
+ 
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: grafana
+      volumes:
+      - emptyDir: {}
+        name: grafana-storage
+      - name: grafana-datasources
+        secret:
+          secretName: grafana-datasources
+      - configMap:
+          name: grafana-dashboards
+        name: grafana-dashboards
+      - configMap:
+          name: grafana-dashboard-apiserver
+        name: grafana-dashboard-apiserver
+      - configMap:
+          name: grafana-dashboard-cluster-total
+        name: grafana-dashboard-cluster-total
+      - configMap:
+          name: grafana-dashboard-controller-manager
+        name: grafana-dashboard-controller-manager
+      - configMap:
+          name: grafana-dashboard-k8s-resources-cluster
+        name: grafana-dashboard-k8s-resources-cluster
+      - configMap:
+          name: grafana-dashboard-k8s-resources-namespace
+        name: grafana-dashboard-k8s-resources-namespace
+      - configMap:
+          name: grafana-dashboard-k8s-resources-node
+        name: grafana-dashboard-k8s-resources-node
+      - configMap:
+          name: grafana-dashboard-k8s-resources-pod
+        name: grafana-dashboard-k8s-resources-pod
+      - configMap:
+          name: grafana-dashboard-k8s-resources-workload
+        name: grafana-dashboard-k8s-resources-workload
+      - configMap:
+          name: grafana-dashboard-k8s-resources-workloads-namespace
+        name: grafana-dashboard-k8s-resources-workloads-namespace
+      - configMap:
+          name: grafana-dashboard-kubelet
+        name: grafana-dashboard-kubelet
+      - configMap:
+          name: grafana-dashboard-namespace-by-pod
+        name: grafana-dashboard-namespace-by-pod
+      - configMap:
+          name: grafana-dashboard-namespace-by-workload
+        name: grafana-dashboard-namespace-by-workload
+      - configMap:
+          name: grafana-dashboard-node-cluster-rsrc-use
+        name: grafana-dashboard-node-cluster-rsrc-use
+      - configMap:
+          name: grafana-dashboard-node-rsrc-use
+        name: grafana-dashboard-node-rsrc-use
+      - configMap:
+          name: grafana-dashboard-nodes
+        name: grafana-dashboard-nodes
+      - configMap:
+          name: grafana-dashboard-persistentvolumesusage
+        name: grafana-dashboard-persistentvolumesusage
+      - configMap:
+          name: grafana-dashboard-pod-total
+        name: grafana-dashboard-pod-total
+      - configMap:
+          name: grafana-dashboard-pods
+        name: grafana-dashboard-pods
+      - configMap:
+          name: grafana-dashboard-prometheus-remote-write
+        name: grafana-dashboard-prometheus-remote-write
+      - configMap:
+          name: grafana-dashboard-prometheus
+        name: grafana-dashboard-prometheus
+      - configMap:
+          name: grafana-dashboard-proxy
+        name: grafana-dashboard-proxy
+      - configMap:
+          name: grafana-dashboard-scheduler
+        name: grafana-dashboard-scheduler
+      - configMap:
+          name: grafana-dashboard-statefulset
+        name: grafana-dashboard-statefulset
+      - configMap:
+          name: grafana-dashboard-workload-total
+        name: grafana-dashboard-workload-total
+      - configMap:
+          name: grafana-dashboard-rocketmq
+        name: grafana-dashboard-rocketmq
+      - configMap:
+          name: grafana-dashboard-mysql
+        name: grafana-dashboard-mysql
--- a/manifests/prometheus/grafana-service.yaml
+++ b/manifests/prometheus/grafana-service.yaml
@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: grafana
+  name: grafana
+  namespace: monitoring
+spec:
+  ports:
+  - name: http
+    nodePort: 23000
+    port: 3000
+    targetPort: http
+  selector:
+    app: grafana
+  type: NodePort
--- a/manifests/prometheus/grafana-serviceAccount.yaml
+++ b/manifests/prometheus/grafana-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana
+  namespace: monitoring
--- a/manifests/prometheus/grafana-serviceMonitor.yaml
+++ b/manifests/prometheus/grafana-serviceMonitor.yaml
@ -0,0 +1,12 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 15s
+    port: http
+  selector:
+    matchLabels:
+      app: grafana
--- a/manifests/prometheus/kube-state-metrics-clusterRole.yaml
+++ b/manifests/prometheus/kube-state-metrics-clusterRole.yaml
@ -0,0 +1,91 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  - nodes
+  - pods
+  - services
+  - resourcequotas
+  - replicationcontrollers
+  - limitranges
+  - persistentvolumeclaims
+  - persistentvolumes
+  - namespaces
+  - endpoints
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  - deployments
+  - replicasets
+  - ingresses
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  - daemonsets
+  - deployments
+  - replicasets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - list
+  - watch
--- a/manifests/prometheus/kube-state-metrics-clusterRoleBinding.yaml
+++ b/manifests/prometheus/kube-state-metrics-clusterRoleBinding.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: monitoring
--- a/manifests/prometheus/kube-state-metrics-deployment.yaml
+++ b/manifests/prometheus/kube-state-metrics-deployment.yaml
@ -0,0 +1,72 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app: kube-state-metrics
+    spec:
+      containers:
+      - args:
+        - --logtostderr
+        - --secure-listen-address=:8443
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:8081/
+        image:  dev-docker-registry.ccyunchina.com/toyangdon/kube-rbac-proxy:v0.4.1
+        name: kube-rbac-proxy-main
+        ports:
+        - containerPort: 8443
+          name: https-main
+        resources:
+          limits:
+            cpu: 20m
+            memory: 40Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+      - args:
+        - --logtostderr
+        - --secure-listen-address=:9443
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:8082/
+        image: dev-docker-registry.ccyunchina.com/toyangdon/kube-rbac-proxy:v0.4.1
+        name: kube-rbac-proxy-self
+        ports:
+        - containerPort: 9443
+          name: https-self
+        resources:
+          limits:
+            cpu: 20m
+            memory: 40Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+      - args:
+        - --host=127.0.0.1
+        - --port=8081
+        - --telemetry-host=127.0.0.1
+        - --telemetry-port=8082
+        image: dev-docker-registry.ccyunchina.com/toyangdon/kube-state-metrics-arm64:v1.9.0
+        name: kube-state-metrics
+        resources:
+          limits:
+            cpu: 100m
+            memory: 150Mi
+          requests:
+            cpu: 100m
+            memory: 150Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: kube-state-metrics
--- a/manifests/prometheus/kube-state-metrics-role.yaml
+++ b/manifests/prometheus/kube-state-metrics-role.yaml
@ -0,0 +1,30 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - extensions
+  resourceNames:
+  - kube-state-metrics
+  resources:
+  - deployments
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - apps
+  resourceNames:
+  - kube-state-metrics
+  resources:
+  - deployments
+  verbs:
+  - get
+  - update
--- a/manifests/prometheus/kube-state-metrics-roleBinding.yaml
+++ b/manifests/prometheus/kube-state-metrics-roleBinding.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kube-state-metrics
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
--- a/manifests/prometheus/kube-state-metrics-service.yaml
+++ b/manifests/prometheus/kube-state-metrics-service.yaml
@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  clusterIP: None
+  ports:
+  - name: https-main
+    port: 8443
+    targetPort: https-main
+  - name: https-self
+    port: 9443
+    targetPort: https-self
+  selector:
+    app: kube-state-metrics
--- a/manifests/prometheus/kube-state-metrics-serviceAccount.yaml
+++ b/manifests/prometheus/kube-state-metrics-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
--- a/manifests/prometheus/kube-state-metrics-serviceMonitor.yaml
+++ b/manifests/prometheus/kube-state-metrics-serviceMonitor.yaml
@ -0,0 +1,30 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    port: https-main
+    relabelings:
+    - action: labeldrop
+      regex: (pod|service|endpoint|namespace)
+    scheme: https
+    scrapeTimeout: 30s
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    port: https-self
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: k8s-app
+  selector:
+    matchLabels:
+      k8s-app: kube-state-metrics
--- a/manifests/prometheus/node-exporter-clusterRole.yaml
+++ b/manifests/prometheus/node-exporter-clusterRole.yaml
@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-exporter
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
--- a/manifests/prometheus/node-exporter-clusterRoleBinding.yaml
+++ b/manifests/prometheus/node-exporter-clusterRoleBinding.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-exporter
+subjects:
+- kind: ServiceAccount
+  name: node-exporter
+  namespace: monitoring
--- a/manifests/prometheus/node-exporter-daemonset.yaml
+++ b/manifests/prometheus/node-exporter-daemonset.yaml
@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: node-exporter
+  name: node-exporter
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app: node-exporter
+  template:
+    metadata:
+      labels:
+        app: node-exporter
+    spec:
+      containers:
+      - args:
+        - --web.listen-address=127.0.0.1:9100
+        - --path.procfs=/host/proc
+        - --path.sysfs=/host/sys
+        - --path.rootfs=/host/root
+        - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/)
+        - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$
+        image:  dev-docker-registry.ccyunchina.com/prom/node-exporter:v0.18.1
+        name: node-exporter
+        resources:
+          limits:
+            cpu: 250m
+            memory: 180Mi
+          requests:
+            cpu: 102m
+            memory: 180Mi
+        volumeMounts:
+        - mountPath: /host/proc
+          name: proc
+          readOnly: false
+        - mountPath: /host/sys
+          name: sys
+          readOnly: false
+        - mountPath: /host/root
+          mountPropagation: HostToContainer
+          name: root
+          readOnly: true
+      - args:
+        - --logtostderr
+        - --secure-listen-address=$(IP):9100
+        - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+        - --upstream=http://127.0.0.1:9100/
+        env:
+        - name: IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: dev-docker-registry.ccyunchina.com/toyangdon/kube-rbac-proxy:v0.4.1
+        name: kube-rbac-proxy
+        ports:
+        - containerPort: 9100
+          hostPort: 9100
+          name: https
+        resources:
+          limits:
+            cpu: 20m
+            memory: 40Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+      hostNetwork: true
+      hostPID: true
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: node-exporter
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /proc
+        name: proc
+      - hostPath:
+          path: /sys
+        name: sys
+      - hostPath:
+          path: /
+        name: root
--- a/manifests/prometheus/node-exporter-service.yaml
+++ b/manifests/prometheus/node-exporter-service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: node-exporter
+  name: node-exporter
+  namespace: monitoring
+spec:
+  clusterIP: None
+  ports:
+  - name: https
+    port: 9100
+    targetPort: https
+  selector:
+    app: node-exporter
--- a/manifests/prometheus/node-exporter-serviceAccount.yaml
+++ b/manifests/prometheus/node-exporter-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-exporter
+  namespace: monitoring
--- a/manifests/prometheus/node-exporter-serviceMonitor.yaml
+++ b/manifests/prometheus/node-exporter-serviceMonitor.yaml
@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: node-exporter
+  name: node-exporter
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    port: https
+    relabelings:
+    - action: replace
+      regex: (.*)
+      replacement: $1
+      sourceLabels:
+      - __meta_kubernetes_pod_node_name
+      targetLabel: instance
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: k8s-app
+  selector:
+    matchLabels:
+      k8s-app: node-exporter
--- a/manifests/prometheus/prometheus-adapter-apiService.yaml
+++ b/manifests/prometheus/prometheus-adapter-apiService.yaml
@ -0,0 +1,13 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: prometheus-adapter
+    namespace: monitoring
+  version: v1beta1
+  versionPriority: 100
--- a/manifests/prometheus/prometheus-adapter-clusterRole.yaml
+++ b/manifests/prometheus/prometheus-adapter-clusterRole.yaml
@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-adapter
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - namespaces
+  - pods
+  - services
+  verbs:
+  - get
+  - list
+  - watch
--- a/manifests/prometheus/prometheus-adapter-clusterRoleAggregatedMetricsReader.yaml
+++ b/manifests/prometheus/prometheus-adapter-clusterRoleAggregatedMetricsReader.yaml
@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: system:aggregated-metrics-reader
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
--- a/manifests/prometheus/prometheus-adapter-clusterRoleBinding.yaml
+++ b/manifests/prometheus/prometheus-adapter-clusterRoleBinding.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-adapter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-adapter
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-adapter-clusterRoleBindingDelegator.yaml
+++ b/manifests/prometheus/prometheus-adapter-clusterRoleBindingDelegator.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: resource-metrics:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-adapter-clusterRoleServerResources.yaml
+++ b/manifests/prometheus/prometheus-adapter-clusterRoleServerResources.yaml
@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: resource-metrics-server-resources
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
--- a/manifests/prometheus/prometheus-adapter-configMap.yaml
+++ b/manifests/prometheus/prometheus-adapter-configMap.yaml
@ -0,0 +1,33 @@
+apiVersion: v1
+data:
+  config.yaml: |
+    resourceRules:
+      cpu:
+        containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>)
+        nodeQuery: sum(1 - rate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            node:
+              resource: node
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        containerLabel: container
+      memory:
+        containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>)
+        nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>)
+        resources:
+          overrides:
+            instance:
+              resource: node
+            namespace:
+              resource: namespace
+            pod:
+              resource: pod
+        containerLabel: container
+      window: 5m
+kind: ConfigMap
+metadata:
+  name: adapter-config
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-adapter-deployment.yaml
+++ b/manifests/prometheus/prometheus-adapter-deployment.yaml
@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-adapter
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: prometheus-adapter
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        name: prometheus-adapter
+    spec:
+      containers:
+      - args:
+        - --cert-dir=/var/run/serving-cert
+        - --config=/etc/adapter/config.yaml
+        - --logtostderr=true
+        - --metrics-relist-interval=1m
+        - --prometheus-url=http://prometheus-k8s.monitoring.svc:9090/
+        - --secure-port=6443
+        image: dev-docker-registry.ccyunchina.com/toyangdon/k8s-prometheus-adapter:v0.5.0
+        name: prometheus-adapter
+        ports:
+        - containerPort: 6443
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmpfs
+          readOnly: false
+        - mountPath: /var/run/serving-cert
+          name: volume-serving-cert
+          readOnly: false
+        - mountPath: /etc/adapter
+          name: config
+          readOnly: false
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: prometheus-adapter
+      volumes:
+      - emptyDir: {}
+        name: tmpfs
+      - emptyDir: {}
+        name: volume-serving-cert
+      - configMap:
+          name: adapter-config
+        name: config
--- a/manifests/prometheus/prometheus-adapter-roleBindingAuthReader.yaml
+++ b/manifests/prometheus/prometheus-adapter-roleBindingAuthReader.yaml
@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: resource-metrics-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: prometheus-adapter
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-adapter-service.yaml
+++ b/manifests/prometheus/prometheus-adapter-service.yaml
@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    name: prometheus-adapter
+  name: prometheus-adapter
+  namespace: monitoring
+spec:
+  ports:
+  - name: https
+    nodePort: 22443
+    port: 443
+    targetPort: 6443
+  selector:
+    name: prometheus-adapter
+  type: NodePort
--- a/manifests/prometheus/prometheus-adapter-serviceAccount.yaml
+++ b/manifests/prometheus/prometheus-adapter-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-adapter
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-clusterRole.yaml
+++ b/manifests/prometheus/prometheus-clusterRole.yaml
@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-k8s
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - endpoints
+  - persistentvolumeclaims
+  - persistentvolumeclaims/status
+  - pods
+  - replicationcontrollers
+  - replicationcontrollers/scale
+  - serviceaccounts
+  - services
+  - services/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/metrics
+  verbs:
+  - get
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
--- a/manifests/prometheus/prometheus-clusterRoleBinding.yaml
+++ b/manifests/prometheus/prometheus-clusterRoleBinding.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-k8s
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-operator-serviceMonitor.yaml
+++ b/manifests/prometheus/prometheus-operator-serviceMonitor.yaml
@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+  namespace: monitoring
+spec:
+  endpoints:
+  - honorLabels: true
+    port: http
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/name: prometheus-operator
+      app.kubernetes.io/version: v0.34.0
--- a/manifests/prometheus/prometheus-prometheus.yaml
+++ b/manifests/prometheus/prometheus-prometheus.yaml
@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  labels:
+    prometheus: k8s
+  name: k8s
+  namespace: monitoring
+spec:
+  alerting:
+    alertmanagers:
+    - name: alertmanager-main
+      namespace: monitoring
+      port: web
+  baseImage: dev-docker-registry.ccyunchina.com/prom/prometheus
+  nodeSelector:
+    kubernetes.io/os: linux
+  podMonitorSelector: {}
+  replicas: 2
+  resources:
+    requests:
+      memory: 400Mi
+  ruleSelector:
+    matchLabels:
+      prometheus: k8s
+      role: alert-rules
+  securityContext:
+    fsGroup: 2000
+    runAsNonRoot: true
+    runAsUser: 1000
+  serviceAccountName: prometheus-k8s
+  serviceMonitorNamespaceSelector: {}
+  serviceMonitorSelector: {}
+  version: v2.11.0
--- a/manifests/prometheus/prometheus-roleBindingConfig.yaml
+++ b/manifests/prometheus/prometheus-roleBindingConfig.yaml
@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s-config
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s-config
+subjects:
+- kind: ServiceAccount
+  name: prometheus-k8s
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-roleBindingSpecificNamespaces.yaml
+++ b/manifests/prometheus/prometheus-roleBindingSpecificNamespaces.yaml
@ -0,0 +1,42 @@
+apiVersion: rbac.authorization.k8s.io/v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: prometheus-k8s
+    namespace: default
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: prometheus-k8s
+  subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: monitoring
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: prometheus-k8s
+    namespace: kube-system
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: prometheus-k8s
+  subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: monitoring
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: prometheus-k8s
+    namespace: monitoring
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: prometheus-k8s
+  subjects:
+  - kind: ServiceAccount
+    name: prometheus-k8s
+    namespace: monitoring
+kind: RoleBindingList
--- a/manifests/prometheus/prometheus-roleConfig.yaml
+++ b/manifests/prometheus/prometheus-roleConfig.yaml
@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s-config
+  namespace: monitoring
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
--- a/manifests/prometheus/prometheus-roleSpecificNamespaces.yaml
+++ b/manifests/prometheus/prometheus-roleSpecificNamespaces.yaml
@ -0,0 +1,51 @@
+apiVersion: rbac.authorization.k8s.io/v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: prometheus-k8s
+    namespace: default
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - services
+    - endpoints
+    - pods
+    verbs:
+    - get
+    - list
+    - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: prometheus-k8s
+    namespace: kube-system
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - services
+    - endpoints
+    - pods
+    verbs:
+    - get
+    - list
+    - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: prometheus-k8s
+    namespace: monitoring
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - services
+    - endpoints
+    - pods
+    verbs:
+    - get
+    - list
+    - watch
+kind: RoleList
--- a/manifests/prometheus/prometheus-rules.yaml
+++ b/manifests/prometheus/prometheus-rules.yaml
--- a/manifests/prometheus/prometheus-service.yaml
+++ b/manifests/prometheus/prometheus-service.yaml
@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    prometheus: k8s
+  name: prometheus-k8s
+  namespace: monitoring
+spec:
+  ports:
+  - name: web
+    nodePort: 22090
+    port: 9090
+    targetPort: web
+  selector:
+    app: prometheus
+    prometheus: k8s
+
+  sessionAffinity: ClientIP
+  type: NodePort
--- a/manifests/prometheus/prometheus-serviceAccount.yaml
+++ b/manifests/prometheus/prometheus-serviceAccount.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus-k8s
+  namespace: monitoring
--- a/manifests/prometheus/prometheus-serviceMonitor.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitor.yaml
@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: prometheus
+  name: prometheus
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: web
+  selector:
+    matchLabels:
+      prometheus: k8s
--- a/manifests/prometheus/prometheus-serviceMonitorApiserver.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitorApiserver.yaml
@ -0,0 +1,37 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: apiserver
+  name: kube-apiserver
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: etcd_(debugging|disk|request|server).*
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: apiserver_admission_controller_admission_latencies_seconds_.*
+      sourceLabels:
+      - __name__
+    - action: drop
+      regex: apiserver_admission_step_admission_latencies_seconds_.*
+      sourceLabels:
+      - __name__
+    port: https
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      serverName: kubernetes
+  jobLabel: component
+  namespaceSelector:
+    matchNames:
+    - default
+  selector:
+    matchLabels:
+      component: apiserver
+      provider: kubernetes
--- a/manifests/prometheus/prometheus-serviceMonitorCoreDNS.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitorCoreDNS.yaml
@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: coredns
+  name: coredns
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 15s
+    port: metrics
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
--- a/manifests/prometheus/prometheus-serviceMonitorKubeControllerManager.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitorKubeControllerManager.yaml
@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-controller-manager
+  name: kube-controller-manager
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: etcd_(debugging|disk|request|server).*
+      sourceLabels:
+      - __name__
+    port: http-metrics
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kube-controller-manager
--- a/manifests/prometheus/prometheus-serviceMonitorKubeScheduler.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitorKubeScheduler.yaml
@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-scheduler
+  name: kube-scheduler
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 30s
+    port: http-metrics
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kube-scheduler
--- a/manifests/prometheus/prometheus-serviceMonitorKubelet.yaml
+++ b/manifests/prometheus/prometheus-serviceMonitorKubelet.yaml
@ -0,0 +1,44 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kubelet
+  name: kubelet
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    port: https-metrics
+    relabelings:
+    - sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    metricRelabelings:
+    - action: drop
+      regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
+      sourceLabels:
+      - __name__
+    path: /metrics/cadvisor
+    port: https-metrics
+    relabelings:
+    - sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  selector:
+    matchLabels:
+      k8s-app: kubelet
--- a/manifests/prometheus/setup/0namespace-namespace.yaml
+++ b/manifests/prometheus/setup/0namespace-namespace.yaml
@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
--- a/manifests/prometheus/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
--- a/manifests/prometheus/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
@ -0,0 +1,239 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: podmonitors.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: PodMonitor
+    plural: podmonitors
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: PodMonitorSpec contains specification parameters for a PodMonitor.
+          properties:
+            jobLabel:
+              description: The label to use to retrieve the job name from.
+              type: string
+            namespaceSelector:
+              description: NamespaceSelector is a selector for selecting either all
+                namespaces or a list of namespaces.
+              properties:
+                any:
+                  description: Boolean describing whether all namespaces are selected
+                    in contrast to a list restricting them.
+                  type: boolean
+                matchNames:
+                  description: List of namespace names.
+                  items:
+                    type: string
+                  type: array
+              type: object
+            podMetricsEndpoints:
+              description: A list of endpoints allowed as part of this PodMonitor.
+              items:
+                description: PodMetricsEndpoint defines a scrapeable endpoint of a
+                  Kubernetes Pod serving Prometheus metrics.
+                properties:
+                  honorLabels:
+                    description: HonorLabels chooses the metric's labels on collisions
+                      with target labels.
+                    type: boolean
+                  honorTimestamps:
+                    description: HonorTimestamps controls whether Prometheus respects
+                      the timestamps present in scraped data.
+                    type: boolean
+                  interval:
+                    description: Interval at which metrics should be scraped
+                    type: string
+                  metricRelabelings:
+                    description: MetricRelabelConfigs to apply to samples before ingestion.
+                    items:
+                      description: 'RelabelConfig allows dynamic rewriting of the
+                        label set, being applied to samples before ingestion. It defines
+                        `<metric_relabel_configs>`-section of Prometheus configuration.
+                        More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
+                      properties:
+                        action:
+                          description: Action to perform based on regex matching.
+                            Default is 'replace'
+                          type: string
+                        modulus:
+                          description: Modulus to take of the hash of the source label
+                            values.
+                          format: int64
+                          type: integer
+                        regex:
+                          description: Regular expression against which the extracted
+                            value is matched. defailt is '(.*)'
+                          type: string
+                        replacement:
+                          description: Replacement value against which a regex replace
+                            is performed if the regular expression matches. Regex
+                            capture groups are available. Default is '$1'
+                          type: string
+                        separator:
+                          description: Separator placed between concatenated source
+                            label values. default is ';'.
+                          type: string
+                        sourceLabels:
+                          description: The source labels select values from existing
+                            labels. Their content is concatenated using the configured
+                            separator and matched against the configured regular expression
+                            for the replace, keep, and drop actions.
+                          items:
+                            type: string
+                          type: array
+                        targetLabel:
+                          description: Label to which the resulting value is written
+                            in a replace action. It is mandatory for replace actions.
+                            Regex capture groups are available.
+                          type: string
+                      type: object
+                    type: array
+                  params:
+                    description: Optional HTTP URL parameters
+                    type: object
+                  path:
+                    description: HTTP path to scrape for metrics.
+                    type: string
+                  port:
+                    description: Name of the port this endpoint refers to. Mutually
+                      exclusive with targetPort.
+                    type: string
+                  proxyUrl:
+                    description: ProxyURL eg http://proxyserver:2195 Directs scrapes
+                      to proxy through this endpoint.
+                    type: string
+                  relabelings:
+                    description: 'RelabelConfigs to apply to samples before ingestion.
+                      More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                    items:
+                      description: 'RelabelConfig allows dynamic rewriting of the
+                        label set, being applied to samples before ingestion. It defines
+                        `<metric_relabel_configs>`-section of Prometheus configuration.
+                        More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
+                      properties:
+                        action:
+                          description: Action to perform based on regex matching.
+                            Default is 'replace'
+                          type: string
+                        modulus:
+                          description: Modulus to take of the hash of the source label
+                            values.
+                          format: int64
+                          type: integer
+                        regex:
+                          description: Regular expression against which the extracted
+                            value is matched. defailt is '(.*)'
+                          type: string
+                        replacement:
+                          description: Replacement value against which a regex replace
+                            is performed if the regular expression matches. Regex
+                            capture groups are available. Default is '$1'
+                          type: string
+                        separator:
+                          description: Separator placed between concatenated source
+                            label values. default is ';'.
+                          type: string
+                        sourceLabels:
+                          description: The source labels select values from existing
+                            labels. Their content is concatenated using the configured
+                            separator and matched against the configured regular expression
+                            for the replace, keep, and drop actions.
+                          items:
+                            type: string
+                          type: array
+                        targetLabel:
+                          description: Label to which the resulting value is written
+                            in a replace action. It is mandatory for replace actions.
+                            Regex capture groups are available.
+                          type: string
+                      type: object
+                    type: array
+                  scheme:
+                    description: HTTP scheme to use for scraping.
+                    type: string
+                  scrapeTimeout:
+                    description: Timeout after which the scrape is ended
+                    type: string
+                  targetPort:
+                    anyOf:
+                    - type: string
+                    - type: integer
+                type: object
+              type: array
+            podTargetLabels:
+              description: PodTargetLabels transfers labels on the Kubernetes Pod
+                onto the target.
+              items:
+                type: string
+              type: array
+            sampleLimit:
+              description: SampleLimit defines per-scrape limit on number of scraped
+                samples that will be accepted.
+              format: int64
+              type: integer
+            selector:
+              description: A label selector is a label query over a set of resources.
+                The result of matchLabels and matchExpressions are ANDed. An empty
+                label selector matches all objects. A null label selector matches
+                no objects.
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements.
+                    The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains
+                      values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies
+                          to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a
+                          set of values. Valid operators are In, NotIn, Exists and
+                          DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator
+                          is In or NotIn, the values array must be non-empty. If the
+                          operator is Exists or DoesNotExist, the values array must
+                          be empty. This array is replaced during a strategic merge
+                          patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - key
+                    - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  description: matchLabels is a map of {key,value} pairs. A single
+                    {key,value} in the matchLabels map is equivalent to an element
+                    of matchExpressions, whose key field is "key", the operator is
+                    "In", and the values array contains only "value". The requirements
+                    are ANDed.
+                  type: object
+              type: object
+          required:
+          - podMetricsEndpoints
+          - selector
+          type: object
+      type: object
+  version: v1
--- a/manifests/prometheus/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
--- a/manifests/prometheus/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
@ -0,0 +1,250 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: prometheusrules.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: PrometheusRule
+    plural: prometheusrules
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          description: ObjectMeta is metadata that all persisted resources must have,
+            which includes all objects users must create.
+          properties:
+            annotations:
+              description: 'Annotations is an unstructured key value map stored with
+                a resource that may be set by external tools to store and retrieve
+                arbitrary metadata. They are not queryable and should be preserved
+                when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations'
+              type: object
+            clusterName:
+              description: The name of the cluster which the object belongs to. This
+                is used to distinguish resources with same name and namespace in different
+                clusters. This field is not set anywhere right now and apiserver is
+                going to ignore it if set in create or update request.
+              type: string
+            creationTimestamp:
+              description: Time is a wrapper around time.Time which supports correct
+                marshaling to YAML and JSON.  Wrappers are provided for many of the
+                factory methods that the time package offers.
+              format: date-time
+              type: string
+            deletionGracePeriodSeconds:
+              description: Number of seconds allowed for this object to gracefully
+                terminate before it will be removed from the system. Only set when
+                deletionTimestamp is also set. May only be shortened. Read-only.
+              format: int64
+              type: integer
+            deletionTimestamp:
+              description: Time is a wrapper around time.Time which supports correct
+                marshaling to YAML and JSON.  Wrappers are provided for many of the
+                factory methods that the time package offers.
+              format: date-time
+              type: string
+            finalizers:
+              description: Must be empty before the object is deleted from the registry.
+                Each entry is an identifier for the responsible component that will
+                remove the entry from the list. If the deletionTimestamp of the object
+                is non-nil, entries in this list can only be removed.
+              items:
+                type: string
+              type: array
+            generateName:
+              description: |-
+                GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.
+
+                If this field is specified and the generated name exists, the server will NOT return a 409 - instead, it will either return 201 Created or 500 with Reason ServerTimeout indicating a unique name could not be found in the time allotted, and the client should retry (optionally after the time indicated in the Retry-After header).
+
+                Applied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency
+              type: string
+            generation:
+              description: A sequence number representing a specific generation of
+                the desired state. Populated by the system. Read-only.
+              format: int64
+              type: integer
+            labels:
+              description: 'Map of string keys and values that can be used to organize
+                and categorize (scope and select) objects. May match selectors of
+                replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels'
+              type: object
+            managedFields:
+              description: ManagedFields maps workflow-id and version to the set of
+                fields that are managed by that workflow. This is mostly for internal
+                housekeeping, and users typically shouldn't need to set or understand
+                this field. A workflow can be the user's name, a controller's name,
+                or the name of a specific apply path like "ci-cd". The set of fields
+                is always in the version that the workflow used when modifying the
+                object.
+              items:
+                description: ManagedFieldsEntry is a workflow-id, a FieldSet and the
+                  group version of the resource that the fieldset applies to.
+                properties:
+                  apiVersion:
+                    description: APIVersion defines the version of this resource that
+                      this field set applies to. The format is "group/version" just
+                      like the top-level APIVersion field. It is necessary to track
+                      the version of a field set because it cannot be automatically
+                      converted.
+                    type: string
+                  fieldsType:
+                    description: 'FieldsType is the discriminator for the different
+                      fields format and version. There is currently only one possible
+                      value: "FieldsV1"'
+                    type: string
+                  fieldsV1:
+                    description: |-
+                      FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.
+
+                      Each key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:<name>', where <name> is the name of a field in a struct, or key in a map 'v:<value>', where <value> is the exact json formatted value of a list item 'i:<index>', where <index> is position of a item in a list 'k:<keys>', where <keys> is a map of  a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.
+
+                      The exact format is defined in sigs.k8s.io/structured-merge-diff
+                    type: object
+                  manager:
+                    description: Manager is an identifier of the workflow managing
+                      these fields.
+                    type: string
+                  operation:
+                    description: Operation is the type of operation which lead to
+                      this ManagedFieldsEntry being created. The only valid values
+                      for this field are 'Apply' and 'Update'.
+                    type: string
+                  time:
+                    description: Time is a wrapper around time.Time which supports
+                      correct marshaling to YAML and JSON.  Wrappers are provided
+                      for many of the factory methods that the time package offers.
+                    format: date-time
+                    type: string
+                type: object
+              type: array
+            name:
+              description: 'Name must be unique within a namespace. Is required when
+                creating resources, although some resources may allow a client to
+                request the generation of an appropriate name automatically. Name
+                is primarily intended for creation idempotence and configuration definition.
+                Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
+              type: string
+            namespace:
+              description: |-
+                Namespace defines the space within each name must be unique. An empty namespace is equivalent to the "default" namespace, but "default" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.
+
+                Must be a DNS_LABEL. Cannot be updated. More info: http://kubernetes.io/docs/user-guide/namespaces
+              type: string
+            ownerReferences:
+              description: List of objects depended by this object. If ALL objects
+                in the list have been deleted, this object will be garbage collected.
+                If this object is managed by a controller, then an entry in this list
+                will point to this controller, with the controller field set to true.
+                There cannot be more than one managing controller.
+              items:
+                description: OwnerReference contains enough information to let you
+                  identify an owning object. An owning object must be in the same
+                  namespace as the dependent, or be cluster-scoped, so there is no
+                  namespace field.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  blockOwnerDeletion:
+                    description: If true, AND if the owner has the "foregroundDeletion"
+                      finalizer, then the owner cannot be deleted from the key-value
+                      store until this reference is removed. Defaults to false. To
+                      set this field, a user needs "delete" permission of the owner,
+                      otherwise 422 (Unprocessable Entity) will be returned.
+                    type: boolean
+                  controller:
+                    description: If true, this reference points to the managing controller.
+                    type: boolean
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids'
+                    type: string
+                required:
+                - apiVersion
+                - kind
+                - name
+                - uid
+                type: object
+              type: array
+            resourceVersion:
+              description: |-
+                An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.
+
+                Populated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+              type: string
+            selfLink:
+              description: |-
+                SelfLink is a URL representing this object. Populated by the system. Read-only.
+
+                DEPRECATED Kubernetes will stop propagating this field in 1.20 release and the field is planned to be removed in 1.21 release.
+              type: string
+            uid:
+              description: |-
+                UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.
+
+                Populated by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids
+              type: string
+          type: object
+        spec:
+          description: PrometheusRuleSpec contains specification parameters for a
+            Rule.
+          properties:
+            groups:
+              description: Content of Prometheus rule file
+              items:
+                description: RuleGroup is a list of sequentially evaluated recording
+                  and alerting rules.
+                properties:
+                  interval:
+                    type: string
+                  name:
+                    type: string
+                  rules:
+                    items:
+                      description: Rule describes an alerting or recording rule.
+                      properties:
+                        alert:
+                          type: string
+                        annotations:
+                          type: object
+                        expr:
+                          anyOf:
+                          - type: string
+                          - type: integer
+                        for:
+                          type: string
+                        labels:
+                          type: object
+                        record:
+                          type: string
+                      required:
+                      - expr
+                      type: object
+                    type: array
+                required:
+                - name
+                - rules
+                type: object
+              type: array
+          type: object
+      type: object
+  version: v1
--- a/manifests/prometheus/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
@ -0,0 +1,346 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: servicemonitors.monitoring.coreos.com
+spec:
+  group: monitoring.coreos.com
+  names:
+    kind: ServiceMonitor
+    plural: servicemonitors
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        spec:
+          description: ServiceMonitorSpec contains specification parameters for a
+            ServiceMonitor.
+          properties:
+            endpoints:
+              description: A list of endpoints allowed as part of this ServiceMonitor.
+              items:
+                description: Endpoint defines a scrapeable endpoint serving Prometheus
+                  metrics.
+                properties:
+                  basicAuth:
+                    description: 'BasicAuth allow an endpoint to authenticate over
+                      basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints'
+                    properties:
+                      password:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                      username:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                    type: object
+                  bearerTokenFile:
+                    description: File to read bearer token for scraping targets.
+                    type: string
+                  bearerTokenSecret:
+                    description: SecretKeySelector selects a key of a Secret.
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be
+                          a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be
+                          defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                  honorLabels:
+                    description: HonorLabels chooses the metric's labels on collisions
+                      with target labels.
+                    type: boolean
+                  honorTimestamps:
+                    description: HonorTimestamps controls whether Prometheus respects
+                      the timestamps present in scraped data.
+                    type: boolean
+                  interval:
+                    description: Interval at which metrics should be scraped
+                    type: string
+                  metricRelabelings:
+                    description: MetricRelabelConfigs to apply to samples before ingestion.
+                    items:
+                      description: 'RelabelConfig allows dynamic rewriting of the
+                        label set, being applied to samples before ingestion. It defines
+                        `<metric_relabel_configs>`-section of Prometheus configuration.
+                        More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
+                      properties:
+                        action:
+                          description: Action to perform based on regex matching.
+                            Default is 'replace'
+                          type: string
+                        modulus:
+                          description: Modulus to take of the hash of the source label
+                            values.
+                          format: int64
+                          type: integer
+                        regex:
+                          description: Regular expression against which the extracted
+                            value is matched. defailt is '(.*)'
+                          type: string
+                        replacement:
+                          description: Replacement value against which a regex replace
+                            is performed if the regular expression matches. Regex
+                            capture groups are available. Default is '$1'
+                          type: string
+                        separator:
+                          description: Separator placed between concatenated source
+                            label values. default is ';'.
+                          type: string
+                        sourceLabels:
+                          description: The source labels select values from existing
+                            labels. Their content is concatenated using the configured
+                            separator and matched against the configured regular expression
+                            for the replace, keep, and drop actions.
+                          items:
+                            type: string
+                          type: array
+                        targetLabel:
+                          description: Label to which the resulting value is written
+                            in a replace action. It is mandatory for replace actions.
+                            Regex capture groups are available.
+                          type: string
+                      type: object
+                    type: array
+                  params:
+                    description: Optional HTTP URL parameters
+                    type: object
+                  path:
+                    description: HTTP path to scrape for metrics.
+                    type: string
+                  port:
+                    description: Name of the service port this endpoint refers to.
+                      Mutually exclusive with targetPort.
+                    type: string
+                  proxyUrl:
+                    description: ProxyURL eg http://proxyserver:2195 Directs scrapes
+                      to proxy through this endpoint.
+                    type: string
+                  relabelings:
+                    description: 'RelabelConfigs to apply to samples before scraping.
+                      More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                    items:
+                      description: 'RelabelConfig allows dynamic rewriting of the
+                        label set, being applied to samples before ingestion. It defines
+                        `<metric_relabel_configs>`-section of Prometheus configuration.
+                        More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
+                      properties:
+                        action:
+                          description: Action to perform based on regex matching.
+                            Default is 'replace'
+                          type: string
+                        modulus:
+                          description: Modulus to take of the hash of the source label
+                            values.
+                          format: int64
+                          type: integer
+                        regex:
+                          description: Regular expression against which the extracted
+                            value is matched. defailt is '(.*)'
+                          type: string
+                        replacement:
+                          description: Replacement value against which a regex replace
+                            is performed if the regular expression matches. Regex
+                            capture groups are available. Default is '$1'
+                          type: string
+                        separator:
+                          description: Separator placed between concatenated source
+                            label values. default is ';'.
+                          type: string
+                        sourceLabels:
+                          description: The source labels select values from existing
+                            labels. Their content is concatenated using the configured
+                            separator and matched against the configured regular expression
+                            for the replace, keep, and drop actions.
+                          items:
+                            type: string
+                          type: array
+                        targetLabel:
+                          description: Label to which the resulting value is written
+                            in a replace action. It is mandatory for replace actions.
+                            Regex capture groups are available.
+                          type: string
+                      type: object
+                    type: array
+                  scheme:
+                    description: HTTP scheme to use for scraping.
+                    type: string
+                  scrapeTimeout:
+                    description: Timeout after which the scrape is ended
+                    type: string
+                  targetPort:
+                    anyOf:
+                    - type: string
+                    - type: integer
+                  tlsConfig:
+                    description: TLSConfig specifies TLS configuration parameters.
+                    properties:
+                      ca: {}
+                      caFile:
+                        description: Path to the CA cert in the Prometheus container
+                          to use for the targets.
+                        type: string
+                      cert: {}
+                      certFile:
+                        description: Path to the client cert file in the Prometheus
+                          container for the targets.
+                        type: string
+                      insecureSkipVerify:
+                        description: Disable target certificate validation.
+                        type: boolean
+                      keyFile:
+                        description: Path to the client key file in the Prometheus
+                          container for the targets.
+                        type: string
+                      keySecret:
+                        description: SecretKeySelector selects a key of a Secret.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must
+                              be a valid secret key.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must
+                              be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                      serverName:
+                        description: Used to verify the hostname for the targets.
+                        type: string
+                    type: object
+                type: object
+              type: array
+            jobLabel:
+              description: The label to use to retrieve the job name from.
+              type: string
+            namespaceSelector:
+              description: NamespaceSelector is a selector for selecting either all
+                namespaces or a list of namespaces.
+              properties:
+                any:
+                  description: Boolean describing whether all namespaces are selected
+                    in contrast to a list restricting them.
+                  type: boolean
+                matchNames:
+                  description: List of namespace names.
+                  items:
+                    type: string
+                  type: array
+              type: object
+            podTargetLabels:
+              description: PodTargetLabels transfers labels on the Kubernetes Pod
+                onto the target.
+              items:
+                type: string
+              type: array
+            sampleLimit:
+              description: SampleLimit defines per-scrape limit on number of scraped
+                samples that will be accepted.
+              format: int64
+              type: integer
+            selector:
+              description: A label selector is a label query over a set of resources.
+                The result of matchLabels and matchExpressions are ANDed. An empty
+                label selector matches all objects. A null label selector matches
+                no objects.
+              properties:
+                matchExpressions:
+                  description: matchExpressions is a list of label selector requirements.
+                    The requirements are ANDed.
+                  items:
+                    description: A label selector requirement is a selector that contains
+                      values, a key, and an operator that relates the key and values.
+                    properties:
+                      key:
+                        description: key is the label key that the selector applies
+                          to.
+                        type: string
+                      operator:
+                        description: operator represents a key's relationship to a
+                          set of values. Valid operators are In, NotIn, Exists and
+                          DoesNotExist.
+                        type: string
+                      values:
+                        description: values is an array of string values. If the operator
+                          is In or NotIn, the values array must be non-empty. If the
+                          operator is Exists or DoesNotExist, the values array must
+                          be empty. This array is replaced during a strategic merge
+                          patch.
+                        items:
+                          type: string
+                        type: array
+                    required:
+                    - key
+                    - operator
+                    type: object
+                  type: array
+                matchLabels:
+                  description: matchLabels is a map of {key,value} pairs. A single
+                    {key,value} in the matchLabels map is equivalent to an element
+                    of matchExpressions, whose key field is "key", the operator is
+                    "In", and the values array contains only "value". The requirements
+                    are ANDed.
+                  type: object
+              type: object
+            targetLabels:
+              description: TargetLabels transfers labels on the Kubernetes Service
+                onto the target.
+              items:
+                type: string
+              type: array
+          required:
+          - endpoints
+          - selector
+          type: object
+      type: object
+  version: v1
--- a/manifests/prometheus/setup/prometheus-operator-clusterRole.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-clusterRole.yaml
@ -0,0 +1,73 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - '*'
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - alertmanagers
+  - prometheuses
+  - prometheuses/finalizers
+  - alertmanagers/finalizers
+  - servicemonitors
+  - podmonitors
+  - prometheusrules
+  verbs:
+  - '*'
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - services/finalizers
+  - endpoints
+  verbs:
+  - get
+  - create
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
--- a/manifests/prometheus/setup/prometheus-operator-clusterRoleBinding.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-clusterRoleBinding.yaml
@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-operator
+subjects:
+- kind: ServiceAccount
+  name: prometheus-operator
+  namespace: monitoring
--- a/manifests/prometheus/setup/prometheus-operator-deployment.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-deployment.yaml
@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      app.kubernetes.io/name: prometheus-operator
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: controller
+        app.kubernetes.io/name: prometheus-operator
+        app.kubernetes.io/version: v0.34.0
+    spec:
+      containers:
+      - args:
+        - --kubelet-service=kube-system/kubelet
+        - --logtostderr=true
+        - --config-reloader-image=dev-docker-registry.ccyunchina.com/toyangdon/configmap-reload-arm64:v0.1
+        - --prometheus-config-reloader=dev-docker-registry.ccyunchina.com/toyangdon/prometheus-config-reloader:v0.34.0
+        image: dev-docker-registry.ccyunchina.com/toyangdon/prometheus-operator:v0.34.0
+        name: prometheus-operator
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          limits:
+            cpu: 200m
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: prometheus-operator
--- a/manifests/prometheus/setup/prometheus-operator-service.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-service.yaml
@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+  namespace: monitoring
+spec:
+  clusterIP: None
+  ports:
+  - name: http
+    port: 8080
+    targetPort: http
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
--- a/manifests/prometheus/setup/prometheus-operator-serviceAccount.yaml
+++ b/manifests/prometheus/setup/prometheus-operator-serviceAccount.yaml
@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/version: v0.34.0
+  name: prometheus-operator
+  namespace: monitoring
--- a/manifests/proprietary-cloud/adapter-api/deployment.yaml
+++ b/manifests/proprietary-cloud/adapter-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-adapter-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-adapter-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-adapter-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-adapter-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-adapter-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-adapter-api-uat:RC-V2.5.20210629.33648.199
+          ports:
+            - name: http
+              containerPort: 6002
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /adapter-api/actuator/health
+              port: 6002
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /adapter-api/actuator/health
+              port: 6002
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-adapter-api
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/adapter-api/service.yaml
+++ b/manifests/proprietary-cloud/adapter-api/service.yaml
@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pumpkin-adapter-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-adapter-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 6002
+  selector:
+    k8s-app: pumpkin-adapter-api
+  type: ClusterIP
--- a/manifests/proprietary-cloud/app-api/deployment.yaml
+++ b/manifests/proprietary-cloud/app-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-app-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-app-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-app-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-app-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-app-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-app-api-uat:RC-V2.5.20210629.33815.197
+          ports:
+            - name: http
+              containerPort: 6009
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /app-api/actuator/health
+              port: 6009
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /app-api/actuator/health
+              port: 6009
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-app-api
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/app-api/service.yaml
+++ b/manifests/proprietary-cloud/app-api/service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: pumpkin-app-api
+  name: pumpkin-app-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 6009
+  selector:
+    k8s-app: pumpkin-app-api
+  type: ClusterIP
--- a/manifests/proprietary-cloud/auth/deployment.yaml
+++ b/manifests/proprietary-cloud/auth/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-auth
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-auth
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-auth
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-auth
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-auth
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-auth-uat:RC-V2.5.20210629.33581.33
+          ports:
+            - name: http
+              containerPort: 8101
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /actuator/health
+              port: 8101
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /actuator/health
+              port: 8101
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-auth
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/auth/service.yaml
+++ b/manifests/proprietary-cloud/auth/service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: pumpkin-auth
+  name: pumpkin-auth
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 8101
+  selector:
+    k8s-app: pumpkin-auth
+  type: ClusterIP
--- a/manifests/proprietary-cloud/baremetal/deployment.yaml
+++ b/manifests/proprietary-cloud/baremetal/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-baremetal
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-baremetal
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-baremetal
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-baremetal
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-baremetal
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-baremetal-uat:RC-V2.5.20210629.33640.58
+          ports:
+            - name: http
+              containerPort: 9888
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /baremetal/actuator/health
+              port: 9888
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /baremetal/actuator/health
+              port: 9888
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-baremetal
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/baremetal/service.yaml
+++ b/manifests/proprietary-cloud/baremetal/service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: pumpkin-baremetal
+  name: pumpkin-baremetal
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 9888
+  selector:
+    k8s-app: pumpkin-baremetal
+  type: ClusterIP
--- a/manifests/proprietary-cloud/bbs-api/deployment.yaml
+++ b/manifests/proprietary-cloud/bbs-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-bbs-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-bbs-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-bbs-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-bbs-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-bbs-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-bbs-api-uat:RC-V2.5.20210629.33583.19
+          ports:
+            - name: http
+              containerPort: 8000
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /bbs-api/actuator/health
+              port: 8000
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /bbs-api/actuator/health
+              port: 8000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-bbs-api
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/bbs-api/service.yaml
+++ b/manifests/proprietary-cloud/bbs-api/service.yaml
@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pumpkin-bbs-api 
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-bbs-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 8000
+  selector:
+    k8s-app: pumpkin-bbs-api
+  type: ClusterIP
--- a/manifests/proprietary-cloud/biz-data.sql
+++ b/manifests/proprietary-cloud/biz-data.sql
--- a/manifests/proprietary-cloud/blog-api/deployment.yaml
+++ b/manifests/proprietary-cloud/blog-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-blog-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-blog-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-blog-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-blog-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-blog-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-blog-api-uat:RC-V2.5.20210629.33584.20
+          ports:
+            - name: http
+              containerPort: 1000
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /blog-api/actuator/health
+              port: 1000
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /blog-api/actuator/health
+              port: 1000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-blog-api
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/blog-api/service.yaml
+++ b/manifests/proprietary-cloud/blog-api/service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: pumpkin-blog-api
+  name: pumpkin-blog-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 1000
+  selector:
+    k8s-app: pumpkin-blog-api
+  type: ClusterIP
--- a/manifests/proprietary-cloud/campaigns-api/deployment.yaml
+++ b/manifests/proprietary-cloud/campaigns-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-campaigns-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-campaigns-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-campaigns-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-campaigns-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-campaigns-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-campaigns-api-uat:RC-V2.5.20210629.35404.40
+          ports:
+            - name: http
+              containerPort: 6008
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '500m'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /campaigns-api/actuator/health
+              port: 6008
+              scheme: HTTP
+            initialDelaySeconds: 160
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 15
+          readinessProbe:
+            httpGet:
+              path: /campaigns-api/actuator/health
+              port: 6008
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-campaigns-api
+                topologyKey: kubernetes.io/hostname
+
--- a/manifests/proprietary-cloud/campaigns-api/service.yaml
+++ b/manifests/proprietary-cloud/campaigns-api/service.yaml
@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: pumpkin-campaigns-api
+  name: pumpkin-campaigns-api
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 6008
+  selector:
+    k8s-app: pumpkin-campaigns-api
+  type: ClusterIP
--- a/manifests/proprietary-cloud/charts-api/deployment.yaml
+++ b/manifests/proprietary-cloud/charts-api/deployment.yaml
@ -0,0 +1,123 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: pumpkin-charts-api
+  namespace: pumpkin-uat
+  labels:
+    k8s-app: pumpkin-charts-api
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: pumpkin-charts-api
+  template:
+    metadata:
+      labels:
+        k8s-app: pumpkin-charts-api
+    spec:
+      volumes:
+        - name: skywalking-agent
+          emptyDir: {}
+      initContainers:
+        - name: agent-container
+          image: dev-docker-registry.ccyunchina.com/apache/skywalking-java-agent:8.10.0-java8
+          command:
+            - /bin/sh
+          args:
+            - '-c'
+            - >-
+              cp -R /skywalking/agent /agent/ && mv -f
+              /agent/agent/optional-plugins/apm-trace-ignore-plugin-8.10.0.jar
+              /agent/agent/plugins/ &&mv -f
+              /agent/agent/optional-plugins/apm-customize-enhance-plugin-8.10.0.jar
+              /agent/agent/plugins/ && rm -f
+              /agent/agent/plugins/apm-lettuce-5.x-plugin-8.10.0.jar
+              /agent/agent/plugins/apm-hikaricp-3.x-4.x-plugin-8.10.0.jar
+          resources: {}
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /agent
+      containers:
+        - name: pumpkin-charts-api
+          image: dev-docker-registry.ccyunchina.com/uat/pumpkin-charts-api-uat:RC-V2.5.20210629.33587.10
+          ports:
+            - name: http
+              containerPort: 6888
+              protocol: TCP
+          env:
+            - name: SPRING_PROFILES_ACTIVE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_CONFIG_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_CONFIG_NAMESPACE
+              value: uat
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_SERVERADDR
+              value: 'nacos:8848'
+            - name: SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE
+              value: uat
+            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
+              value: 'prod-skywalking-oap.kube-public:11800'
+            - name: SW_LOGGING_DIR
+              value: /app/logs
+            - name: SW_SERVICE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: 'metadata.labels[''k8s-app'']'
+            - name: SW_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SW_AGENT_NAME
+              value: '${SW_NAMESPACE}::${SW_SERVICE_NAME}'
+            - name: JAVA_OPTIONS
+              value: >-
+                -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+PrintCommandLineFlags
+                -XX:MaxRAMFraction=1 -Xms3g  -XX:MetaspaceSize=200m
+                -javaagent:/skywalking/agent/skywalking-agent.jar
+                -Dskywalking.trace.ignore_path=/*/actuator/health
+                -Dskywalking.plugin.jdbc.trace_sql_parameters=true
+          resources:
+            limits:
+              cpu: '4'
+              memory: 4Gi
+            requests:
+              cpu: '1'
+              memory: 1Gi
+          volumeMounts:
+            - name: skywalking-agent
+              mountPath: /skywalking
+          livenessProbe:
+            httpGet:
+              path: /charts-api/actuator/health
+              port: 6888
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 3
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /charts-api/actuator/health
+              port: 6888
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 9
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: k8s-app
+                      operator: In
+                      values:
+                        - pumpkin-charts-api
+                topologyKey: kubernetes.io/hostname
+
--- a/Show More
+++ b/Show More