init

LICENSE

@@ -0,0 +1,208 @@
+Apache License
+
+Version 2.0, January 2004
+
+http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION,
+AND DISTRIBUTION
+
+   1. Definitions.
+
+      
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution
+as defined by Sections 1 through 9 of this document.
+
+      
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+      
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct
+or indirect, to cause the direction or management of such entity, whether
+by contract or otherwise, or (ii) ownership of fifty percent (50%) or more
+of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+      
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions
+granted by this License.
+
+      
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+      
+
+"Object" form shall mean any form resulting from mechanical transformation
+or translation of a Source form, including but not limited to compiled object
+code, generated documentation, and conversions to other media types.
+
+      
+
+"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that
+is included in or attached to the work (an example is provided in the Appendix
+below).
+
+      
+
+"Derivative Works" shall mean any work, whether in Source or Object form,
+that is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative
+Works shall not include works that remain separable from, or merely link (or
+bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+      
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative
+Works thereof, that is intentionally submitted to Licensor for inclusion in
+the Work by the copyright owner or by an individual or Legal Entity authorized
+to submit on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication
+sent to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor
+for the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+      
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently incorporated
+within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable copyright license to reproduce, prepare
+Derivative Works of, publicly display, publicly perform, sublicense, and distribute
+the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License,
+each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section) patent
+license to make, have made, use, offer to sell, sell, import, and otherwise
+transfer the Work, where such license applies only to those patent claims
+licensable by such Contributor that are necessarily infringed by their Contribution(s)
+alone or by combination of their Contribution(s) with the Work to which such
+Contribution(s) was submitted. If You institute patent litigation against
+any entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that the Work or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses granted to You
+under this License for that Work shall terminate as of the date such litigation
+is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+Derivative Works thereof in any medium, with or without modifications, and
+in Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a copy
+of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating that
+You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source
+form of the Work, excluding those notices that do not pertain to any part
+of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution,
+then any Derivative Works that You distribute must include a readable copy
+of the attribution notices contained within such NOTICE file, excluding those
+notices that do not pertain to any part of the Derivative Works, in at least
+one of the following places: within a NOTICE text file distributed as part
+of the Derivative Works; within the Source form or documentation, if provided
+along with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works
+that You distribute, alongside or as an addendum to the NOTICE text from the
+Work, provided that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction,
+or distribution of Your modifications, or for any such Derivative Works as
+a whole, provided Your use, reproduction, and distribution of the Work otherwise
+complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+Contribution intentionally submitted for inclusion in the Work by You to the
+Licensor shall be under the terms and conditions of this License, without
+any additional terms or conditions. Notwithstanding the above, nothing herein
+shall supersede or modify the terms of any separate license agreement you
+may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names,
+trademarks, service marks, or product names of the Licensor, except as required
+for reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to
+in writing, Licensor provides the Work (and each Contributor provides its
+Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied, including, without limitation, any warranties
+or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR
+A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness
+of using or redistributing the Work and assume any risks associated with Your
+exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether
+in tort (including negligence), contract, or otherwise, unless required by
+applicable law (such as deliberate and grossly negligent acts) or agreed to
+in writing, shall any Contributor be liable to You for damages, including
+any direct, indirect, special, incidental, or consequential damages of any
+character arising as a result of this License or out of the use or inability
+to use the Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all other commercial
+damages or losses), even if such Contributor has been advised of the possibility
+of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work
+or Derivative Works thereof, You may choose to offer, and charge a fee for,
+acceptance of support, warranty, indemnity, or other liability obligations
+and/or rights consistent with this License. However, in accepting such obligations,
+You may act only on Your own behalf and on Your sole responsibility, not on
+behalf of any other Contributor, and only if You agree to indemnify, defend,
+and hold each Contributor harmless for any liability incurred by, or claims
+asserted against, such Contributor by reason of your accepting any such warranty
+or additional liability. END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own identifying
+information. (Don't include the brackets!) The text should be enclosed in
+the appropriate comment syntax for the file format. We also recommend that
+a file or class name and description of purpose be included on the same "printed
+page" as the copyright notice for easier identification within third-party
+archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+
+distributed under the License is distributed on an "AS IS" BASIS,
+
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+See the License for the specific language governing permissions and
+
+limitations under the License.

README.md

@@ -1,2 +1,98 @@
-# ccyunchina-deploy
+# 说明
+*部署脚本于长城专有云平台*  
+部署脚本中包括了kubernetes底层组件、efk等一系列服务的安装。采用ansbile脚本实现自动安装，运维人员需要对ansible工具有一定简单了解。主要服务基本实现全容器化、k8s化部署，可以通过kubernetes dashboard监控到所有容器服务；部署脚本基于centos 7 或者kylin v10 sp1，要求内核版本为4以上；既提供一键快速安装方式，也提供分步执行安装方式。
+# 组件版本
+| 名称                    | 版本号       | 备注     |
+|-------------------------|--------------|----------|
+| Kernel                  | 4以上        |          |
+| kube-apiserver          | 1.21.0       |          |
+| kube-controller-manager | 1.21.0       |          |
+| kube-scheduler          | 1.21.0       |          |
+| kube-proxy              | 1.21.0       |          |
+| kubelet                 | 1.21.0       |          |
+| etcd                    | 3.3.15       |          |
+| calico                  | 3.3.1        |          |
+| docker                  | 18.06.3-ce   |          |
+| coredns                 | 1.6.2        |          |
+| kubernets-dashboard     | 1.10.1       |          |
+| traefik                 | 2.1.1        |          |
+| pause                   | 3.1          |          |
+| elasticsearch           | 6.2.4        |          |
+| keepalived              | 2.0.19-r0    |          |
+| Haproxy                 | 2.1.2        |          |
+| gluster                 | 4.0       |          |
+| heketi                  | 6.0          |          |
+| metrics-server          | 0.2.0       |          |
+| node-problem-detector   | 0.4.1       |          |
+| openvpn                 | 2.1          |          |
+| efk                 | 7.10.2          |          |
+| mysql                 | 8.0          |          |
+| redis                 | 5.0          |          |
+| rocketmq                 | 4.5.0          |          |
+| gw-proprietary-cloud                 | 0.1          |          |
 
+# 部署示意图
+![k8s部署图](https://github.com/toyangdon/k8s_deploy/blob/master/kubernetes%20%E7%BB%84%E7%BB%87%E5%9B%BE.png?raw=true)
+# 快速安装
+1. 安装ansible
+`yum install -y ansible`
+2. 下载部署文件到部署节点的/etc/ansible目录下  
+`git clone –depth=1 https://git.trustie.net/toyangdon/proprietary-cloud-deploy.git`  
+将部署文件复制到/etc/ansible目录下  
+`cp -rf k8s_deploy/* /etc/ansible/`  
+3. 配置集群安装信息  
+**根据实际情况修改`hosts`文件**  
+`vi /etc/ansible/hosts`  
+4. 配置ssh免密码  
+`sh tools/ssh-key-copy.sh root ${passwd} #请输入实际的root用户密码`  
+5. 执行一键安装  
+**centos**   
+`ansible-playbook setup.yml`  
+**kylin v10 sp1**  
+`ansible-playbook -e 'ansible_python_interpreter=/usr/bin/python3.7' setup.yml`  
+**单机部署**  
+`ansible-playbook -e 'apiserver_mem_requests=100Mi'  -e 'apiserver_cpu_requests=100m'  setup.yml`  
+# 分步安装
+`playbooks`目录提供分步安装的相关playbook，主要分为两大块`kubernetes`和`gpaas`
+## `kubernetes` 部署
+1. `ansible-playbook playbooks/kubernetes/00.check.yml` 检查集群服务器
+1. `ansible-playbook playbooks/kubernetes/01.docker.yml` 在所有主机上安装并启动docker服务
+2. `ansible-playbook playbooks/kubernetes/02.prepare.yml` 服务器通用配置，生成并分发集群所需相关证书
+3. `ansible-playbook playbooks/kubernetes/03.harbor.yml` 部署harbor节点，安装并启动harbor服务（可选）
+4. `ansible-playbook playbooks/kubernetes/04.lb.yml` 准备lb节点所需的相关安装文件，包括keepalived和haproxy
+5. `ansible-playbook playbooks/kubernetes/05.kube-master.yml` 准备master节点所需的相关安装文件
+6. `ansible-playbook playbooks/kubernetes/06.kube-node.yml` 在主机上安装并启动kubelet服务，先启动lb,再启动master，最后启动kube-node
+7. `ansible-playbook playbooks/kubernetes/07.calico.yml` 在主机上准备calico服务所需要的相关安装文件（与flannel可选）
+8. `ansible-playbook playbooks/kubernetes/07.flannel.yml` 在主机上准备flannel服务所需要的相关安装文件（与calico可选） （暂时不可用）
+9. `ansible-playbook playbooks/kubernetes/09.storage-nfs.yml` 安装nfs服务（与gfs可选）（暂时不可用）
+10. `ansible-playbook playbooks/kubernetes/10.storage-gluster.yml` 准备安装gfs服务  
+11. `ansible-playbook playbooks/kubernetes/20.addnode.yml` 新增节点
+12. `ansible-playbook playbooks/kubernetes/30.addons.yml` kubernetes所有插件服务的部署，包括kube-proxy、kubedns、calico、glusterfs等等
+13. `ansible-playbook playbooks/kubernetes/90.setup.yml` 一键安装kubernetes,即顺序执行以上所有步骤（除了20.addnode）
+14. `ansible-playbook playbooks/kubernetes/99.clean.yml` 一键清理kubernetes集群（慎用）
+
+## `gpass` 部署
+目前分为`efk`和`monitor`二部分
+
+### `efk` 部署
+1. `ansible-playbook playbooks/gpaas/elk/01.es.yml` es部署
+3. `ansible-playbook playbooks/gpaas/elk/02.fluentd.yml` fluentd
+4. `ansible-playbook playbooks/gpaas/elk/03.kibana.yml` kibana部署
+5. `ansible-playbook playbooks/gpaas/elk/90.setup.yml` 一键安装elk，即顺序执行以上所有步骤  
+
+### `monitor` 部署
+1. `ansible-playbook playbooks/gpaas/monitor/01.prometheus.yml` prometheus部署
+2. `ansible-playbook playbooks/gpaas/monitor/90.setup.yml` 一键安装监控平台，即顺序执行以上所有步骤
+
+### 一键部署`gpass`
+1. `ansible-playbook playbooks/gpaas/90.setup.yml`
+
+## `专有云组件`部署
+1. `ansible-playbook playbooks/proprietary-cloud/90.setup.yml`
+
+# `ansible`容器化
+1. 安装docker   
+`systemctl stop firewalld` #关闭防火墙  
+`sh tools/docker/install.sh` 
+2. 运行ansible镜像执行k8s安装   
+`/opt/k8s/bin/docker run --name ansible -it --privileged -v /root/k8s-install:/etc/ansible toyangdon/ansible:latest` #进入容器内执行ansible脚本

dockerfiles/ansible/Dockerfile

@@ -0,0 +1,9 @@
+FROM centos
+RUN yum install centos-release-ansible-29 -y && yum install ansible openssh-server openssh-clients -y
+RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' && ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N "" && ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" &&\
+    ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa && cat /root/.ssh/id_rsa.pub >/root/.ssh/authorized_keys
+RUN chmod +x -R /usr/bin/
+
+ENV LANG C.UTF-8
+
+RUN echo "/usr/sbin/sshd" >>/root/.bash_profile

dockerfiles/es/es-index-clean/Dockerfile

@@ -0,0 +1,6 @@
+FROM centos:7
+MAINTAINER yangdong <toyangdon@163.com>
+WORKDIR /home/gwcloud
+COPY es_index_clean_config clean.sh /home/gwcloud/
+RUN chmod +x /home/gwcloud/clean.sh
+ENTRYPOINT sh /home/gwcloud/clean.sh

dockerfiles/es/es-index-clean/es_index_clean_config

@@ -0,0 +1,10 @@
+applog 3
+filebeat 7
+metricbeat 3
+elasticsearch 7
+gfs 3
+harbor 3
+k8s 3
+mongodb 3
+rabbitmq 3
+zipkin 3

example/hosts.m-masters.example

@@ -0,0 +1,69 @@
+[deploy]
+localhost
+
+#镜像仓库
+[registry]
+10.2.1.212
+
+#节点配置
+#管理节点 
+[kube-master]
+10.2.1.212
+10.2.1.170
+10.2.1.148
+
+#计算节点
+[kube-compute-node]
+
+#存储节点
+[kube-storage-node]
+10.2.1.212
+10.2.1.170
+
+[kube-node:children]
+kube-compute-node
+kube-storage-node
+
+[kube-cluster:children]
+kube-node
+kube-master
+lb
+
+
+#代理节点
+#多管理节点时必须配置有代理节点
+#LB_ROLE：master、backup、lb_only，master\backup表示使用keepalived实现haproxy高可用,lb_only表示不使用keepalived
+[lb]
+10.2.1.212
+10.2.1.170
+
+[elasticsearch]
+10.2.1.212
+10.2.1.170
+
+[mysql]
+10.2.1.148
+
+# 预留组，后续添加node节点使用
+[new-node]
+
+[kube-cluster:vars]
+ansible_python_interpreter=/usr/bin/python3
+
+[all:vars]
+#集群 MASTER IP  如果单master部署，则该值为master ip，如果多master部署,则改值为lb的vip
+MASTER_IP="10.2.1.164"
+
+#gfs磁盘设备名
+gfs_device=/dev/sdb
+
+zstackServerIP=10.1.1.9
+zstackServerAdminUser=admin
+zstackServerAdminPassword=Greatwall@123
+
+#默认区域
+defaultRegionCode=cn-cs-1
+defaultRegionName=长沙一区
+
+#专有云访问域名
+gwcloudDomain=gwcloud.com

group_vars/all

@@ -0,0 +1,107 @@
+# ---------集群主要参数---------------
+MASTER_PORT: 6443
+
+#TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
+BOOTSTRAP_TOKEN: "0a3417df589e82d359ff6fc3d3c90fc0"
+
+# 集群网络插件，目前支持calico
+CLUSTER_NETWORK: "calico"
+
+# 服务网段 (Service CIDR），部署前路由不可达，部署后集群内使用 IP:Port 可达
+SERVICE_CIDR: "10.68.0.0/16"
+
+# POD 网段 (Cluster CIDR），部署前路由不可达，**部署后**路由可达
+CLUSTER_CIDR: "172.20.0.0/16"
+
+# 服务端口范围 (NodePort Range)
+NODE_PORT_RANGE: "10000-32767"
+
+# kubernetes 服务 IP (预分配，一般是 SERVICE_CIDR 中第一个IP)
+CLUSTER_KUBERNETES_SVC_IP: "10.68.0.1"
+
+# 集群 DNS 服务 IP (从 SERVICE_CIDR 中预分配)
+CLUSTER_DNS_SVC_IP: "10.68.0.2"
+
+# 集群 DNS 域名
+CLUSTER_DNS_DOMAIN: "cluster.local."
+
+#默认二进制文件目录
+bin_dir: "/opt/k8s/bin"
+
+#集群安装目录
+cluster_dir: "/etc/kubernetes"
+
+#证书目录
+ca_dir: "{{cluster_dir}}/ssl"
+
+#部署目录，即 ansible 工作目录
+base_dir: "/etc/ansible"
+
+#数据目录
+data_dir: "/data"
+
+# 基础镜像
+k8s_pod_infra_container_image: toyangdon/pause-arm64:3.4.1
+
+#存储卷类型
+storage_type: glusterfs
+
+#默认主机名，要求唯一，默认规则：主机组名-主机ip最后一段 ，如果hosts文件中不是配置的ip(有可能是域名)，那么就直接取域名
+NODE_ID: >-
+  node-{{inventory_hostname.split('.')[3]}}
+
+#资源限制
+#生产配置
+#KUBE_RESERVED: "{'cpu':'200m','memory':'500Mi','ephemeral-storage':'1Gi'}"
+system_reserved: >-
+  {'cpu': '{{ ansible_processor_vcpus * 1000 * 0.08 }}m','memory': '{{ansible_memtotal_mb * 0.05}}Mi','ephemeral-storage':'500Mi'}
+kube_reserved: >-
+  {'cpu': '{{ansible_processor_vcpus * 1000 * 0.08}}m','memory': '{{ansible_memtotal_mb * 0.05}}Mi','ephemeral-storage':'500Mi'}
+eviction_hard: >-
+  {'memory.available': '{{ansible_memtotal_mb * 0.05}}Mi','nodefs.available':'5%','imagefs.available':'15%','nodefs.inodesFree': '5%'}
+
+node_labels: >-
+  {%- if NODE_LABELS is defined -%}{{NODE_LABELS}},{%- endif -%}
+  {%- for group_name in group_names|reject("match","kube-cluster|kube-node")  -%}node.kubernetes.io/{{group_name|replace("kube-","")}}=true{%- if not loop.last-%},{%- endif-%}{%- endfor -%}
+  
+node_taints: ""
+
+#本地镜像仓库端口
+registry_port: 6550
+
+BASE_IMAGE_URL: >-
+  {{ groups['registry'][0]}}:{{registry_port}}
+
+#docker insecure registry 如果有多个用逗号分开
+docker_insecure_registry: >-
+  {{BASE_IMAGE_URL}}
+
+#是否部署glusterfs
+deploy_gfs: >-
+  {%- if groups['kube-storage-node']|length !=0 -%}true{%- else -%}false{%- endif -%}
+
+#是否部署监控告警
+deploy_monitor: true
+  
+#是否离线
+deploy_offline: true
+
+#是否部署本地镜像仓库
+deploy_docker_registry: true
+
+#是否加载镜像
+load_images: true
+
+#是否推送镜像
+push_images: true
+
+
+#工具镜像（证书）
+gw_tools_image: toyangdon/gw-tools:1.1
+
+#证书生成命令
+#cfssl_cmd: "{{bin_dir}}/docker run --rm -v {{ ca_dir }}:/workdir {{ gw_tools_image }} sh -c"
+cfssl_cmd: "cd {{ ca_dir }} && export PATH=$PATH:{{bin_dir}} && sh -c"
+
+#专有云业务镜像仓库
+CLOUD_IMAGE_URL: "{{BASE_IMAGE_URL}}"

group_vars/elasticsearch

@@ -0,0 +1,11 @@
+# es运行的jvm参数，两个值建议设置相同，并且为虚拟机内存的一半
+#es_jvm_xms_size: "-Xms6g"
+#es_jvm_xmx_size: "-Xmx6g"
+
+es_storage_capacity: auto
+
+#es_cpu_limit: 4000m
+#es_cpu_requests: 1000m
+
+#es_mem_limit: 7Gi 
+#es_mem_requests: 1Gi

group_vars/harbor

@@ -0,0 +1,14 @@
+harbor_admin_password: Harbor12345
+
+harbor_ext_endpoint: >-
+  {{ inventory_hostname }}
+
+images_src_path: >-
+  {{base_dir}}/images
+  
+images_path: /data/tmp/images
+  
+images_file_name: images.tar.gz
+
+images_file: >-
+  {{ images_path }}/{{ images_file_name }}

group_vars/kube-cluster

@@ -0,0 +1 @@
+

group_vars/kube-master

@@ -0,0 +1,7 @@
+secure_port: 6444
+
+#保留值
+system_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}
+kube_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}

group_vars/kube-storage-node

@@ -0,0 +1,10 @@
+node_labels: >-
+  {%- if NODE_LABELS is defined -%}{{NODE_LABELS}},{%- endif -%}
+  {%- for group_name in group_names|reject("match","kube-cluster|kube-node")  -%}node.kubernetes.io/{{group_name|replace("kube-","")}}=true{%- if not loop.last-%},{%- endif-%}{%- endfor -%}
+  {%- if groups["kube-storage-node"][0] == inventory_hostname -%},node.kubernetes.io/heketi=true{%- endif -%}
+
+#保留值
+system_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}
+kube_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}

group_vars/lb

@@ -0,0 +1,17 @@
+# 区分多个instance的VRRP组播，同网段不能重复，取值在0-255之间
+VIRTUAL_ROUTER_ID: >-
+  {{MASTER_IP.split('.')[3]}}
+  
+#lb节点部署方式 process/container
+lb_type: container
+
+LB_ROLE: >-
+  {%- if groups["lb"][0] == inventory_hostname -%}master
+  {%- else -%}backup
+  {%- endif -%}
+
+#保留值
+system_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}
+kube_reserved: >-
+  {'cpu': '500m','memory': '1024Mi','ephemeral-storage':'500Mi'}

group_vars/logstash

@@ -0,0 +1,10 @@
+#logstash_jvm_xms_size: -Xms6g
+#logstash_jvm_xmx_size: -Xmx6g
+
+logstash_type: container
+
+#logstash_cpu_limit: 4000m
+#logstash_cpu_requests: 1000m
+
+#logstash_mem_limit: 7Gi 
+#logstash_mem_requests: 1Gi

group_vars/new-node

@@ -0,0 +1,9 @@
+node_labels: >-
+  {%- if NODE_LABELS is defined -%}{{NODE_LABELS}},{%- endif -%}node.kubernetes.io/kube-compute-node=true
+  
+node_taints: >-
+  {%- if NODE_TAINTS is defined -%}{{NODE_TAINTS}}{%- endif -%}
+  
+#默认主机名，要求唯一，默认规则：主机组名-主机ip最后一段 ，如果hosts文件中不是配置的ip(有可能是域名)，那么就直接取域名
+NODE_ID: >-
+  kube-compute-node-{{inventory_hostname.split('.')[3]}}

group_vars/registry

@@ -0,0 +1,9 @@
+images_src_path: >-
+  {{base_dir}}/images
+
+images_path: /data/tmp/images
+
+images_file_name: images.tar.gz
+
+images_file: >-
+  {{ images_path }}/{{ images_file_name }}

hosts

@@ -0,0 +1,82 @@
+[deploy]
+localhost
+
+#镜像仓库
+[registry]
+10.2.1.212
+
+#节点配置
+#管理节点 
+[kube-master]
+10.2.1.212
+10.2.1.170
+10.2.1.148
+
+#计算节点
+[kube-compute-node]
+
+#存储节点
+[kube-storage-node]
+10.2.1.212
+10.2.1.170
+
+[kube-node:children]
+kube-compute-node
+kube-storage-node
+
+[kube-cluster:children]
+kube-node
+kube-master
+lb
+
+
+#代理节点
+#多管理节点时必须配置有代理节点
+#LB_ROLE：master、backup、lb_only，master\backup表示使用keepalived实现haproxy高可用,lb_only表示不使用keepalived
+[lb]
+10.2.1.212
+10.2.1.170
+
+[elasticsearch]
+10.2.1.212
+10.2.1.170
+
+[mysql]
+10.2.1.148
+10.2.1.170
+
+#openvpn服务，请另外创建一台虚拟机（4c8g）作为openvpn服务器，该虚拟机可用于暴露私有云内部网络，openvpnExternalIp表示服务器的外部IP，openvpnRouteCIDR表示需求经过vpn路由的ip范围
+[openvpn]
+10.2.1.224  openvpnExternalIp=10.2.1.224 openvpnRouteCIDR=10.2.1.0/16 ansible_ssh_user=root ansible_ssh_pass=Greatwall@123
+
+# 预留组，后续添加node节点使用
+[new-node]
+
+[kube-cluster:vars]
+ansible_python_interpreter=/usr/bin/python3
+
+[all:vars]
+#集群 MASTER IP  如果单master部署，则该值为master ip，如果多master部署,则改值为lb的vip
+MASTER_IP="10.2.1.164"
+
+#gfs磁盘设备名
+gfs_device=/dev/sdb
+
+zstackServerIP=10.1.1.9
+zstackServerAdminUser=admin
+zstackServerAdminPassword=Greatwall@123
+
+#默认区域
+defaultRegionCode=cn-cs-1
+defaultRegionName=长沙一区
+
+#专有云访问域名
+gwcloudDomain=gwcloud.com
+
+#专有云告警邮箱服务器
+monitorEmailEnable=true
+monitorEmailHost=mail.greatwall.com.cn
+monitorEmailPort=465
+monitorEmailUsername=hnck
+monitorEmailFrom=hnck@greatwall.com.cn
+monitorEmailPassword=Eghucn-91406

package/Jenkinsfile_for_package

@@ -0,0 +1,81 @@
+def projectProperties = [
+        [$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', numToKeepStr: '5']],
+        parameters([
+            string(name: 'BRANCH_NAME', defaultValue: 'master', description: '代码分支'),
+            string(name: 'CLEAN_FLAG', defaultValue: "false", description: '是否清理工作空间'),
+            string(name: 'VERSION', defaultValue: "1.0", description: '包版本'),
+            string(name: 'GENERATE_IMAGES_LISTS', defaultValue: "true", description: '是否生成images_list文件'),
+            string(name: 'INCLUDE_APPS_IMAGES', defaultValue: "true", description: '是否包含应用镜像'),
+            string(name: 'DEST_PATH_PREFIX', defaultValue: "/var/jenkins_home", description: '打包文件保存目录'),
+            string(name: 'HARBOR_SRC', defaultValue: 'docker.io', description: '源镜像仓库')
+        ])
+]
+
+properties(projectProperties)
+
+node('master') {
+
+        //部署文件路径
+        DEPLOY_PROJECT_PATH="${WORKSPACE}/k8s_deploy";
+
+        //镜像文件保存目录
+        IMAGES_PATH="${DEPLOY_PROJECT_PATH}/images"
+
+        //基础镜像文件列表
+        IMAGES_LIST="${DEPLOY_PROJECT_PATH}/tools/images/images_list"
+
+        //应用商店镜像文件列表
+        APPS_IMAGES_LIST="${DEPLOY_PROJECT_PATH}/tools/images/apps_images_list"
+
+        //打包文件保存目录
+        DEST_PATH="${DEST_PATH_PREFIX}/k8s_data/${VERSION}"
+
+        //源harobr镜像库地址
+        HARBOR_SRC="docker.io"
+
+
+        stage('拉取k8s部署文件') {
+            if("${CLEAN_FLAG}" == "true"){
+                        cleanWs()
+                }
+                dir(DEPLOY_PROJECT_PATH){
+                        git branch: "${BRANCH_NAME}", url: 'https://git.trustie.net/toyangdon/proprietary-cloud-deploy.git'
+                }
+        }
+
+        if( "${GENERATE_IMAGES_LISTS}" == "true" ){
+                stage('生成基础镜像列表'){
+                        sh "grep '_image:' -hr ${DEPLOY_PROJECT_PATH}/roles ${DEPLOY_PROJECT_PATH}/group_vars|cut -d':' -f2,3|sed  's/\"//g'|sed 's/ //g' |sort -u >${IMAGES_LIST} "
+                }
+        }
+
+        stage('拉取并打包基础镜像') {
+            sh "docker login -u toyangdon -pyd880309"
+            sh "apt install pigz -y"
+            if( "${INCLUDE_APPS_IMAGES}" == "true" ){
+                sh "cat ${APPS_IMAGES_LIST} >> ${IMAGES_LIST}"
+                sh "rm -f ${APPS_IMAGES_LIST}"
+            }
+            sh "sh ${DEPLOY_PROJECT_PATH}/tools/images/images_batch_save.sh ${IMAGES_PATH} ${HARBOR_SRC} ${IMAGES_LIST}"
+
+        }
+        
+        stage('拉取并导出ansible镜像') {
+            sh "docker pull toyangdon/ansible:latest"
+            sh "docker save toyangdon/ansible:latest -o k8s_deploy/ansible_image.tar"
+        }
+
+
+
+        stage('打包部署文件') {
+                sh "rm -rf k8s_deploy/.git"
+                sh "tar --use-compress-program=pigz -cvpf k8s_deploy.tar.gz k8s_deploy"
+        }
+
+        stage('归档部署文件'){
+                //根据版本号创建目录，如果目录已经存在则报错退出
+                sh "mkdir -p ${DEST_PATH}"
+                sh "cp k8s_deploy.tar.gz ${DEST_PATH}/"
+        }
+}
+

package/image_push/image_list

@@ -0,0 +1,8 @@
+dev-docker-registry.ccyunchina.com/uat/system-management:v0.3.beta-20210907.1467.45 toyangdon/system-management:v0.3
+dev-docker-registry.ccyunchina.com/uat/cloud-service:v0.3.beta-20210907.1460.31 toyangdon/cloud-service:v0.3
+dev-docker-registry.ccyunchina.com/uat/trade-service:v0.3.beta-20210907.1460.43 toyangdon/trade-service:v0.3
+dev-docker-registry.ccyunchina.com/uat/file-service:v0.3.beta-20210907.496.20 toyangdon/file-service:v0.3
+dev-docker-registry.ccyunchina.com/uat/cloud-base:v0.3.beta-20210907.1462.55 toyangdon/cloud-base:v0.3
+dev-docker-registry.ccyunchina.com/uat/vue-cloud:v0.3.beta-20210907.1414.63 toyangdon/vue-cloud:v0.3
+dev-docker-registry.ccyunchina.com/uat/vue-admin:v0.3.beta-20210907.1415.70 toyangdon/vue-admin:v0.3
+dev-docker-registry.ccyunchina.com/dev/activiti-service:v0.3.beta-20210907.1470.16 toyangdon/activiti-service:v0.3

package/image_push/push.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -x
+
+image_list_file=image_list
+#docker_registry_desc=dockerhub.ccyunchina.com
+#docker_registry_desc=registry.cn-hangzhou.aliyuncs.com
+docker_registry_desc=docker.io
+
+#docker login $docker_registry_desc -u 153307747@qq.com -pyd880309
+docker login $docker_registry_desc -u toyangdon -pyd880309
+if [ $? != 0 ];then
+	echo "error:docker registry login fail! please check username or password is correct"
+	exit 1
+  fi
+
+while read line
+do      
+  src_image=`echo $line |awk '{print $1}'`
+  desc_image=${docker_registry_desc}/`echo $line |awk '{print $2}'`
+  docker pull $src_image
+  if [ $? != 0 ];then
+	echo "error:pull image ${src_image} fail! please check image name is correct"
+	exit 1
+  fi
+  docker tag $src_image $desc_image
+  docker push $desc_image
+  if [ $? != 0 ];then
+	echo "error:push image ${image} fail!"
+	exit 1
+  fi
+  echo "------------------$name push success!-------------"  
+done < $image_list_file
+docker logout
+
+echo "------------------all images push success-------------" 

package/jenkins.yml

@@ -0,0 +1,126 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: jenkins
+spec:
+  accessModes:
+  - ReadWriteMany
+  capacity:
+    storage: 30Gi
+  local:
+    path: /data/jenkins
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - "kube-compute-node-240"
+  persistentVolumeReclaimPolicy: Delete
+  storageClassName: local-storage
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: jenkins
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 30Gi
+  volumeMode: Filesystem
+  storageClassName: local-storage
+---
+# Source: jenkins/templates/svc.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: jenkins
+  labels:
+    app: jenkins
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: "Cluster"
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+    - name: https
+      port: 443
+      targetPort: https
+  selector:
+    app: jenkins
+---
+# Source: jenkins/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jenkins
+  labels:
+    app: jenkins
+spec:
+  selector:
+    matchLabels:
+      app: jenkins
+  template:
+    metadata:
+      labels:
+        app: jenkins
+    spec:      
+      containers:
+        - name: jenkins
+          image: calmmopey/jenkins-arm64-2.277.4:fixed
+          env:
+            - name: JAVA_OPTS
+              value: "-Xmx3072m -Xms512m -XX:ErrorFile=/var/jenkins_home/hs_err_pid<pid>.log"
+          ports:
+            - name: http
+              containerPort: 8080
+            - name: https
+              containerPort: 8443
+          livenessProbe:
+            httpGet:
+              path: /login
+              port: http
+            initialDelaySeconds: 300
+            periodSeconds: 10
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 10
+          readinessProbe:
+            httpGet:
+              path: /login
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 3
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+            limits:
+              cpu: 2
+              memory: 4Gi 
+            requests:
+              cpu: 300m
+              memory: 512Mi
+          volumeMounts:
+            - name: jenkins-data
+              mountPath: /var/jenkins_home
+            - name: docker-sock
+              mountPath: /var/run/docker.sock
+            - name: docker
+              mountPath: /bin/docker
+      volumes:
+        - name: jenkins-data
+          persistentVolumeClaim:
+            claimName: jenkins
+        - name: docker-sock
+          hostPath:
+            path: /var/run/docker.sock
+            type: ''
+        - name: docker
+          hostPath:
+            path: /opt/k8s/bin/docker
+            type: ''

playbooks/gpaas/90.setup.yml

@@ -0,0 +1,3 @@
+- import_playbook: efk/90.setup.yml
+
+- import_playbook: monitor/90.setup.yml

playbooks/gpaas/efk/01.es.yml

@@ -0,0 +1,4 @@
+- hosts:
+  - elasticsearch
+  roles:
+  - gpaas/efk/elasticsearch

playbooks/gpaas/efk/03.fluentd.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/efk/fluentd

playbooks/gpaas/efk/04.kibana.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/efk/kibana

playbooks/gpaas/efk/90.setup.yml

@@ -0,0 +1,7 @@
+- import_playbook: 01.es.yml
+
+#- import_playbook: 02.logstash.yml
+
+- import_playbook: 03.fluentd.yml
+
+- import_playbook: 04.kibana.yml

playbooks/gpaas/monitor/01.prometheus.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/metricbeat

playbooks/gpaas/monitor/02.metricbeat.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/metricbeat

playbooks/gpaas/monitor/03.grafana.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/grafana

playbooks/gpaas/monitor/04.alert.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/alert

playbooks/gpaas/monitor/05.inspect.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/inspect

playbooks/gpaas/monitor/06.push.yml

@@ -0,0 +1,4 @@
+# 插件部署
+- hosts: deploy
+  roles:
+  - gpaas/monitor/push

playbooks/gpaas/monitor/90.setup.yml

@@ -0,0 +1,14 @@
+#- import_playbook: ../elk/01.es.yml
+ # when: deploy_es == 'true'
+
+#- import_playbook: 02.metricbeat.yml
+
+#- import_playbook: 03.grafana.yml
+
+#- import_playbook: 04.alert.yml
+
+#- import_playbook: 05.inspect.yml
+
+#- import_playbook: 06.push.yml
+
+- import_playbook:  monitor.yml