diff --git a/sql/update_20240423.sql b/sql/update_20240423.sql new file mode 100644 index 0000000..8e5e66e --- /dev/null +++ b/sql/update_20240423.sql @@ -0,0 +1,6 @@ +ALTER TABLE `sync_repo_mapping` +ADD COLUMN `inter_token` VARCHAR(100) COMMENT '内部仓库token', +ADD COLUMN `exter_token` VARCHAR(100) COMMENT '外部仓库token'; + +ALTER TABLE `sync_branch_mapping` +MODIFY COLUMN `sync_direction` enum('to_outer', 'to_inter') COMMENT '首次同步方向'; diff --git a/src/api/Sync_config.py b/src/api/Sync_config.py index 232c693..fab0704 100644 --- a/src/api/Sync_config.py +++ b/src/api/Sync_config.py @@ -14,7 +14,7 @@ from src.utils.sync_log import sync_log, LogType, api_log from src.api.Controller import APIController as Controller from src.router import SYNC_CONFIG as router from src.do.sync_config import SyncDirect -from src.dto.sync_config import SyncRepoDTO, SyncBranchDTO, LogDTO +from src.dto.sync_config import SyncRepoDTO, SyncBranchDTO, LogDTO, ModifyRepoDTO from src.service.sync_config import SyncService, LogService from src.service.cronjob import sync_repo_task, sync_branch_task from src.base.status_code import Status, SYNCResponse, SYNCException @@ -213,6 +213,19 @@ class SyncDirection(Controller): msg=data.status_msg ) + @router.put("/repo/{repo_name}/repo_addr", response_model=SYNCResponse, description='更新仓库地址') + async def update_repo_addr( + self, request: Request, user: str = Depends(user), + repo_name: str = Path(..., description="仓库名称"), + dto: ModifyRepoDTO = Body(..., description="更新仓库地址信息") + ): + api_log(LogType.INFO, f"用户 {user} 使用 PUT 方法访问接口 {request.url.path} ", user) + data = await self.service.update_repo_addr(repo_name=repo_name, dto=dto) + return SYNCResponse( + code_status=data.code_status, + msg=data.status_msg + ) + @router.put("/repo/{repo_name}", response_model=SYNCResponse, description='更新仓库同步状态') async def update_repo_status( self, request: Request, user: str = Depends(user), diff --git a/src/do/sync_config.py b/src/do/sync_config.py index 5d0111c..efc2bbc 100644 --- a/src/do/sync_config.py +++ b/src/do/sync_config.py @@ -28,7 +28,9 @@ class SyncRepoMapping(DataObject): repo_name = Column(String(128), unique=True, nullable=False, comment="仓库名称") enable = Column(Boolean, default=True, comment="是否启用同步") internal_repo_address = Column(String, nullable=False, comment="内部仓库地址") + inter_token = Column(String, nullable=True, comment="内部仓库token") external_repo_address = Column(String, nullable=False, comment="外部仓库地址") + exter_token = Column(String, nullable=True, comment="外部仓库token") sync_granularity = Column(Enum(SyncType), comment="同步类型") sync_direction = Column(Enum(SyncDirect), comment="首次同步方向") created_at = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP'), comment="创建时间") diff --git a/src/dto/sync_config.py b/src/dto/sync_config.py index 6b3f4ac..a0acef3 100644 --- a/src/dto/sync_config.py +++ b/src/dto/sync_config.py @@ -6,11 +6,20 @@ class SyncRepoDTO(BaseModel): repo_name: str = Field(..., description="仓库名称") enable: bool = Field(..., description="同步状态") internal_repo_address: str = Field(..., description="内部仓库地址") + inter_token: str = Field(None, description="内部仓库token") external_repo_address: str = Field(..., description="外部仓库地址") + exter_token: str = Field(None, description="外部仓库token") sync_granularity: int = Field(..., description="1 为仓库粒度的同步, 2 为分支粒度的同步") sync_direction: int = Field(..., description="1 表示内部仓库同步到外部, 2 表示外部仓库同步到内部") +class ModifyRepoDTO(BaseModel): + internal_repo_address: str = Field(None, description="内部仓库地址") + inter_token: str = Field(None, description="内部仓库token") + external_repo_address: str = Field(None, description="外部仓库地址") + exter_token: str = Field(None, description="外部仓库token") + + class SyncBranchDTO(BaseModel): enable: bool = Field(..., description="是否启用分支同步") internal_branch_name: str = Field(..., description="内部仓库分支名") diff --git a/src/service/cronjob.py b/src/service/cronjob.py index 4fb9f8b..c7d2f8c 100644 --- a/src/service/cronjob.py +++ b/src/service/cronjob.py @@ -25,9 +25,33 @@ def get_git_error(stderr: str): return GITMSGException(Status.UNKNOWN_ERROR) +def get_repo_address_with_token(address: str, token: str) -> str: + if token is None or token == "": + return address + try: + if not address.startswith('https') and not address.startswith('http'): + raise Exception('address is error') + + if address.startswith('https'): + owner_name = address[8:].split("/")[1] + return address[:8] + owner_name + ":" + token + '@' + address[8:] + elif address.startswith('http'): + owner_name = address[7:].split("/")[1] + return address[:7] + owner_name + ":" + token + '@' + address[7:] + except Exception as e: + print(e) + + def shell(cmd, dire: str, log_name: str, user: str): log = f'Execute cmd: ' + cmd - sync_log(LogType.INFO, log, log_name, user) + if 'git clone' in log: + sync_log(LogType.INFO, 'Execute cmd: git clone', log_name, user) + elif 'git remote add' in log: + sync_log(LogType.INFO, 'Execute cmd: git remote add', log_name, user) + elif 'git ls-remote' in log: + sync_log(LogType.INFO, '获取仓库分支信息', log_name, user) + else: + sync_log(LogType.INFO, log, log_name, user) output = subprocess.run(shlex.split(cmd), cwd=dire, capture_output=True, text=True) if output.returncode != 0: git_error = get_git_error(output.stderr) @@ -42,17 +66,18 @@ def init_repos(repo, log_name: str, user: str): repo_dir = os.path.join(SYNC_DIR, repo.repo_name) if not os.path.exists(repo_dir): sync_log(LogType.INFO, "初始化仓库 *********", log_name, user) - repo_name = repo.repo_name + inter_repo_addr = get_repo_address_with_token(repo.internal_repo_address, repo.inter_token) + exter_repo_addr = get_repo_address_with_token(repo.external_repo_address, repo.exter_token) if repo.sync_direction == SyncDirect.to_outer: # 克隆内部仓库到同步目录下 - shell(f'git clone -b master {repo.internal_repo_address} {repo_dir}', SYNC_DIR, log_name, user) + shell(f'git clone -b master {inter_repo_addr} {repo_dir}', SYNC_DIR, log_name, user) else: # 克隆外部仓库到同步目录下 - shell(f'git clone -b master {repo.external_repo_address} {repo_dir}', SYNC_DIR, log_name, user) + shell(f'git clone -b master {exter_repo_addr} {repo_dir}', SYNC_DIR, log_name, user) # 添加internal远程仓库，并强制使用 - shell(f'git remote add -f internal {repo.internal_repo_address}', repo_dir, log_name, user) + shell(f'git remote add -f internal {inter_repo_addr}', repo_dir, log_name, user) # 添加external远程仓库，并强制使用 - shell(f'git remote add -f external {repo.external_repo_address}', repo_dir, log_name, user) + shell(f'git remote add -f external {exter_repo_addr}', repo_dir, log_name, user) def inter_to_outer(repo, branch, log_name: str, user: str): @@ -97,32 +122,35 @@ async def sync_repo_task(repo, user): log_name = f'sync_{repo.repo_name}.log' init_repos(repo, log_name, user) sync_log(LogType.INFO, f'************ 执行{repo.repo_name}仓库同步 ************', log_name, user) - if repo.sync_direction == SyncDirect.to_outer: - stm = shell(f"git ls-remote --heads {repo.internal_repo_address}", SYNC_DIR, log_name, user) - branch_list_output = stm.stdout.split('\n') - for branch in branch_list_output: - if branch: - branch_name = branch.split('/')[-1].strip() - branch = SyncBranchDTO(enable=1, internal_branch_name=branch_name, external_branch_name=branch_name) - sync_log(LogType.INFO, f'Execute inter to outer {branch_name} branch Sync', log_name, user) - inter_to_outer(repo, branch, log_name, user) - else: - stm = shell(f"git ls-remote --heads {repo.external_repo_address}", SYNC_DIR, log_name, user) - branch_list_output = stm.stdout.split('\n') - for branch in branch_list_output: - if branch: - branch_name = branch.split('/')[-1].strip() - branch = SyncBranchDTO(enable=1, internal_branch_name=branch_name, external_branch_name=branch_name) - sync_log(LogType.INFO, f'Execute outer to inter {branch_name} branch Sync', log_name, user) - outer_to_inter(repo, branch, log_name, user) + try: + if repo.sync_direction == SyncDirect.to_outer: + inter_repo_addr = get_repo_address_with_token(repo.internal_repo_address, repo.inter_token) + stm = shell(f"git ls-remote --heads {inter_repo_addr}", SYNC_DIR, log_name, user) + branch_list_output = stm.stdout.split('\n') + for branch in branch_list_output: + if branch: + branch_name = branch.split('/')[-1].strip() + branch = SyncBranchDTO(enable=1, internal_branch_name=branch_name, external_branch_name=branch_name) + sync_log(LogType.INFO, f'Execute inter to outer {branch_name} branch Sync', log_name, user) + inter_to_outer(repo, branch, log_name, user) + else: + exter_repo_addr = get_repo_address_with_token(repo.external_repo_address, repo.exter_token) + stm = shell(f"git ls-remote --heads {exter_repo_addr}", SYNC_DIR, log_name, user) + branch_list_output = stm.stdout.split('\n') + for branch in branch_list_output: + if branch: + branch_name = branch.split('/')[-1].strip() + branch = SyncBranchDTO(enable=1, internal_branch_name=branch_name, external_branch_name=branch_name) + sync_log(LogType.INFO, f'Execute outer to inter {branch_name} branch Sync', log_name, user) + outer_to_inter(repo, branch, log_name, user) + sync_log(LogType.INFO, f'************ {repo.repo_name}仓库同步完成 ************', log_name, user) + finally: + if config.DELETE_SYNC_DIR: + os.path.exists(SYNC_DIR) and os.removedirs(SYNC_DIR) + sync_log(LogType.INFO, f'删除同步工作目录: {SYNC_DIR}', log_name, user) - if config.DELETE_SYNC_DIR: - os.path.exists(SYNC_DIR) and os.removedirs(SYNC_DIR) - sync_log(LogType.INFO, f'删除同步工作目录: {SYNC_DIR}', log_name, user) - - sync_log(LogType.INFO, f'************ {repo.repo_name}仓库同步完成 ************', log_name, user) - await log_service.insert_repo_log(repo_name=repo.repo_name, direct=repo.sync_direction) - os.remove(os.path.join(log_path, log_name)) + await log_service.insert_repo_log(repo_name=repo.repo_name, direct=repo.sync_direction) + os.remove(os.path.join(log_path, log_name)) async def sync_branch_task(repo, branches, direct, user): @@ -131,18 +159,20 @@ async def sync_branch_task(repo, branches, direct, user): log_name = f'sync_{repo.repo_name}_{branch.id}.log' init_repos(repo, log_name, user) sync_log(LogType.INFO, f'************ 执行分支同步 ************', log_name, user) - if direct == SyncDirect.to_inter: - sync_log(LogType.INFO, f'Execute outer to inter {branch.external_branch_name} branch Sync', log_name, user) - commit_id = outer_to_inter(repo, branch, log_name, user) - else: - sync_log(LogType.INFO, f'Execute inter to outer {branch.internal_branch_name} branch Sync', log_name, user) - commit_id = inter_to_outer(repo, branch, log_name, user) + commit_id = '' + try: + if direct == SyncDirect.to_inter: + sync_log(LogType.INFO, f'Execute outer to inter {branch.external_branch_name} branch Sync', log_name, user) + commit_id = outer_to_inter(repo, branch, log_name, user) + else: + sync_log(LogType.INFO, f'Execute inter to outer {branch.internal_branch_name} branch Sync', log_name, user) + commit_id = inter_to_outer(repo, branch, log_name, user) + sync_log(LogType.INFO, f'************ 分支同步完成 ************', log_name, user) + finally: + if config.DELETE_SYNC_DIR: + os.path.exists(SYNC_DIR) and os.removedirs(SYNC_DIR) + sync_log(LogType.INFO, f'删除同步工作目录: {SYNC_DIR}', log_name, user) - if config.DELETE_SYNC_DIR: - os.path.exists(SYNC_DIR) and os.removedirs(SYNC_DIR) - sync_log(LogType.INFO, f'删除同步工作目录: {SYNC_DIR}', log_name, user) - - sync_log(LogType.INFO, f'************ 分支同步完成 ************', log_name, user) - await log_service.insert_branch_log(repo.repo_name, direct, branch.id, commit_id) - os.remove(os.path.join(log_path, log_name)) + await log_service.insert_branch_log(repo.repo_name, direct, branch.id, commit_id) + os.remove(os.path.join(log_path, log_name)) diff --git a/src/service/sync_config.py b/src/service/sync_config.py index e51f28c..ee98b7b 100644 --- a/src/service/sync_config.py +++ b/src/service/sync_config.py @@ -1,8 +1,9 @@ import re from typing import List, Union, Optional, Dict from .service import Service +from src.utils import base from src.dao.sync_config import SyncBranchDAO, SyncRepoDAO, LogDAO -from src.dto.sync_config import SyncBranchDTO, SyncRepoDTO, RepoDTO, AllRepoDTO, GetBranchDTO, LogDTO, BranchDTO +from src.dto.sync_config import SyncBranchDTO, SyncRepoDTO, RepoDTO, AllRepoDTO, GetBranchDTO, LogDTO, BranchDTO, ModifyRepoDTO from src.do.sync_config import SyncDirect, SyncType from src.base.status_code import Status, SYNCException from src.utils.sync_log import log_path @@ -105,6 +106,26 @@ class SyncService(Service): return SYNCException(Status.SUCCESS) + async def update_repo_addr(self, repo_name: str, dto: ModifyRepoDTO) -> SYNCException: + repo = await self.sync_repo_dao.get(repo_name=repo_name) + if repo is None: + return SYNCException(Status.REPO_NOTFOUND) + update_fields = {} + if dto.internal_repo_address is not None: + if not base.check_addr(dto.internal_repo_address): + return SYNCException(Status.REPO_ADDR_ILLEGAL) + update_fields['internal_repo_address'] = dto.internal_repo_address + if dto.external_repo_address is not None: + if not base.check_addr(dto.external_repo_address): + return SYNCException(Status.REPO_ADDR_ILLEGAL) + update_fields['external_repo_address'] = dto.external_repo_address + if dto.inter_token is not None: + update_fields['inter_token'] = dto.inter_token + if dto.exter_token is not None: + update_fields['exter_token'] = dto.exter_token + await self.sync_repo_dao.update(repo, **update_fields) + return SYNCException(Status.SUCCESS) + async def update_repo(self, repo_name: str, enable: bool) -> SYNCException: repo = await self.sync_repo_dao.get(repo_name=repo_name) if repo is None: diff --git a/src/utils/base.py b/src/utils/base.py index 878f941..a0eda46 100644 --- a/src/utils/base.py +++ b/src/utils/base.py @@ -1,12 +1,15 @@ import re -GIT_HTTP_PATTERN = r'https://.*.com/(.*)/(.*).git' +GIT_HTTPS_PATTERN = r'https://.*.com/(.*)/(.*).git' +GIT_HTTP_PATTERN = r'http://.*.com/(.*)/(.*).git' GIT_SSH_PATTERN = r'git@.*.com:(.*)/(.*).git' def check_addr(repo_address: str) -> bool: try: if repo_address.startswith('https'): + pattern = GIT_HTTPS_PATTERN + elif repo_address.startswith('http'): pattern = GIT_HTTP_PATTERN else: pattern = GIT_SSH_PATTERN