新增 kafka 集群配置模式

common/src/main/java/cn/org/gitlink/notification/common/utils/ThreadUtil.java

@@ -3,7 +3,6 @@ package cn.org.gitlink.notification.common.utils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;

executor/src/main/resources/application.properties

@@ -1,22 +1,22 @@
 server.port=8083
 logging.config=classpath:logback.xml
 ### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
-xxl.job.admin.addresses=http://127.0.0.1:9999/xxl-job-admin
+xxl.job.admin.addresses=http://localhost:9999/xxl-job-admin
 ### xxl-job, access token
 xxl.job.accessToken=
 ### xxl-job executor appname
-xxl.job.executor.appname=xxl-job-executor-trustie-sample
+xxl.job.executor.appname=gitlink-notification-executor-trustie-sample
 ### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
 xxl.job.executor.address=
 ### xxl-job executor server-info
 xxl.job.executor.ip=
 xxl.job.executor.port=82
 ### xxl-job executor log-path
-xxl.job.executor.logpath=./log/xxl-job/jobhandler
+xxl.job.executor.logpath=./logs/xxl-job/jobhandler
 ### xxl-job executor log-retention-days
 xxl.job.executor.logretentiondays=30
 
-kafka.consumer.servers=gitlink_kafka:9092
+kafka.consumer.servers=localhost:29092
 kafka.consumer.group.id=0
 kafka.consumer.auto.offset.reset=earliest
 kafka.consumer.enable.auto.commit=false

middleware/.env.example

@@ -13,17 +13,21 @@ MYSQL_CONTAINER_NAME=gitlink_mysql
 REDIS_CONTAINER_NAME=gitlink_redis
 XXL_JOB_ADMIN_CONTAINER_NAME=gitlink_xxl_job_admin
 ZOOKEEPER_CONTAINER_NAME=gitlink_zookeepr
-KAFKA_CONTAINER_NAME=gitlink_kafka
+KAFKA_CONTAINER_01_NAME=gitlink_kafka_01
-KAFKA_MANAGER_CONTAINER_NAME=gitlink_kafka_manager
+KAFKA_CONTAINER_02_NAME=gitlink_kafka_02
 
 #============================================ Ports config ============================================
 
-MYSQL_LOCAL_PORT=33306          # TODO: MysQL 本地端口
+MYSQL_LOCAL_PORT=33306
-REDIS_LOCAL_PORT=6379           # TODO: Redis 本地端口
+REDIS_LOCAL_PORT=6379
-ZOOKEEPER_LOCAL_PORT=2181       # TODO: Zookeeper 本地端口
+ZOOKEEPER_LOCAL_PORT=22181
-KAFKA_LOCAL_PORT=9092           # TODO: Kafka 本地端口
+XXL_JOB_ADMIN_LOCAL_PORT=9999
-KAFKA_MANAGER_LOCAL_PORT=9003   # TODO: Kafka 管理本地端口
+KAFKA_01_LOCAL_PORT=29092
-XXL_JOB_ADMIN_LOCAL_PORT=9999   #TODO: xxl-job-admin 本地端口
+KAFKA_02_LOCAL_PORT=39092
+
+#============================================ Kafka Broker ID config ============================================
+KAFKA_01_BROKER_ID=1
+KAFKA_02_BROKER_ID=2
 
 #============================================ Path config ============================================
 # TODO: 需要预执行的 sql 脚本目录
@@ -37,11 +41,3 @@ MYSQL_ROOT_PASSWORD=gitLiNk_827
 MYSQL_USER=gitlink
 MYSQL_PASSWORD=giTlinK0^827
 MYSQL_DATABASE=gitlink_notification
-
-#============================================ Kafaka config ============================================
-KAFKA_LOG_RETENTION_HOURS=120
-KAFKA_MESSAGE_MAX_BYTES=10000000
-KAFKA_REPLICA_FETCH_MAX_BYTES=10000000
-KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS=60000
-KAFKA_NUM_PARTITIONS=3
-KAFKA_DELETE_RETENTION_MS=1000

middleware/services.yml

@@ -4,6 +4,7 @@ services:
   mysql:
     image: mysql:${MYSQL_VERSION}
     container_name: ${MYSQL_CONTAINER_NAME}
+    hostname: mysql
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
       - MYSQL_USER=${MYSQL_USER}
@@ -20,10 +21,10 @@ services:
       - gitlink_network
 
 
-
   redis:
     image: redis:${REDIS_VERSION}
     container_name: ${REDIS_CONTAINER_NAME}
+    hostname: redis
     volumes:
       - ${DOCKER_DATA_PATH}/redis/data:/data
       - ${DOCKER_DATA_PATH}/redis/logs:/logs
@@ -33,10 +34,10 @@ services:
       - gitlink_network
 
 
-
   xxl-job-admin:
     image: xuxueli/xxl-job-admin:${XXL_JOB_ADMIN_VERSION}
     container_name: ${XXL_JOB_ADMIN_CONTAINER_NAME}
+    hostname: xxl_job_admin
     environment:
       - JAVA_OPTS=-Xmx512m
       - PARAMS=--spring.datasource.url=jdbc:mysql://${MYSQL_CONTAINER_NAME}:3306/gitlink_notification?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai --spring.datasource.username=${MYSQL_USER} --spring.datasource.password=${MYSQL_PASSWORD}
@@ -49,51 +50,57 @@ services:
     networks:
       - gitlink_network
 
-
+  # See Also: https://hub.docker.com/r/confluentinc/cp-zookeeper
-
   zookeeper:
-    image: wurstmeister/zookeeper
+    image: confluentinc/cp-zookeeper:latest
     container_name: ${ZOOKEEPER_CONTAINER_NAME}
+    hostname: zookeeper
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
     ports:
       - ${ZOOKEEPER_LOCAL_PORT}:2181
+    networks:
+      - gitlink_network
+
+  # See Also: https://hub.docker.com/r/confluentinc/cp-kafka
+  kafka1:
+    image: confluentinc/cp-kafka:latest
+    container_name: ${KAFKA_CONTAINER_01_NAME}
+    hostname: kafka1
     depends_on:
-      - kafka
+      - zookeeper
-    networks:
-      - gitlink_network
-
-
-
-  kafka:
-    image: wurstmeister/kafka
-    container_name: ${KAFKA_CONTAINER_NAME}
-    volumes:
-      - /etc/localtime:/etc/localtime
     ports:
-      - ${KAFKA_LOCAL_PORT}:9092
+      - ${KAFKA_01_LOCAL_PORT}:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    networks:
+      - gitlink_network
+
+  # See Also: https://hub.docker.com/r/confluentinc/cp-kafka
+  kafka2:
+    image: confluentinc/cp-kafka:latest
+    container_name: ${KAFKA_CONTAINER_02_NAME}
+    hostname: kafka2
+    depends_on:
+      - zookeeper
+    ports:
+      - ${KAFKA_02_LOCAL_PORT}:39092
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:9092,PLAINTEXT_HOST://localhost:39092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
     networks:
       - gitlink_network
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: ${KAFKA_CONTAINER_NAME}
-      KAFKA_ZOOKEEPER_CONNECT: ${ZOOKEEPER_CONTAINER_NAME}:2181
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_LOG_RETENTION_HOURS: ${KAFKA_LOG_RETENTION_HOURS}
-      KAFKA_MESSAGE_MAX_BYTES: ${KAFKA_MESSAGE_MAX_BYTES}
-      KAFKA_REPLICA_FETCH_MAX_BYTES: ${KAFKA_REPLICA_FETCH_MAX_BYTES}
-      KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: ${KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS}
-      KAFKA_NUM_PARTITIONS: ${KAFKA_NUM_PARTITIONS}
-      KAFKA_DELETE_RETENTION_MS: ${KAFKA_DELETE_RETENTION_MS}
 
-#  kafka-manager:
-#    image: sheepkiller/kafka-manager
-#    container_name: ${KAFKA_MANAGER_CONTAINER_NAME}
-#    networks:
-#      - gitlink_network
-#    depends_on:
-#      - kafka
-#    environment:
-#      ZK_HOSTS: ${ZOOKEEPER_CONTAINER_NAME}:2181
-#    ports:
-#      - ${KAFKA_MANAGER_LOCAL_PORT}:9000
 
 networks:
   gitlink_network: