This commit is contained in:
李政谦 2024-06-24 19:53:56 +08:00
parent d906ee6c55
commit 1c287fd837
1 changed files with 161 additions and 110 deletions

271
sync.py
View File

@ -23,8 +23,10 @@ from src.common.repo import Repo, RepoType
async def apply_diff(project, job, pull: PullRequestDTO, dir): async def apply_diff(project, job, pull: PullRequestDTO, dir):
# 从GitHub地址中提取组织和仓库名称
organization, repo = github.transfer_github_to_name(project.github_address) organization, repo = github.transfer_github_to_name(project.github_address)
# 根据仓库类型构建基础URL
baseUrl = "" baseUrl = ""
if pull.type == RepoType.Github: if pull.type == RepoType.Github:
baseUrl = f"{config.GITHUB_ENV['github_api_diff_address']}/{organization}/{repo}/pull/" baseUrl = f"{config.GITHUB_ENV['github_api_diff_address']}/{organization}/{repo}/pull/"
@ -33,9 +35,11 @@ async def apply_diff(project, job, pull: PullRequestDTO, dir):
elif pull.type == RepoType.Gitcode: elif pull.type == RepoType.Gitcode:
pass pass
# 构建diff文件的URL和临时文件路径
diffUrl = baseUrl + str(pull.id) + ".diff" diffUrl = baseUrl + str(pull.id) + ".diff"
tmpfile = dir + "/" + str(pull.id) + "_diff" tmpfile = dir + "/" + str(pull.id) + "_diff"
# 根据仓库类型构建下载diff文件的命令
download_diff_cmd = "" download_diff_cmd = ""
if pull.type == RepoType.Github: if pull.type == RepoType.Github:
download_diff_cmd = f"curl -X GET {diffUrl} -H 'Accept: application/vnd.github.v3.diff'" download_diff_cmd = f"curl -X GET {diffUrl} -H 'Accept: application/vnd.github.v3.diff'"
@ -44,117 +48,144 @@ async def apply_diff(project, job, pull: PullRequestDTO, dir):
elif pull.type == RepoType.Gitcode: elif pull.type == RepoType.Gitcode:
pass pass
# 下载diff文件并写入临时文件中
with open(tmpfile, "w") as diff_file: with open(tmpfile, "w") as diff_file:
diff, err = await cmd.shell(download_diff_cmd, dir, job) diff, err = await cmd.shell(download_diff_cmd, dir, job)
diff_file.write(diff) diff_file.write(diff)
# git apply --check first # 检查diff是否可以无错误地应用
out, err = await cmd.shell('git apply --check ' + tmpfile, dir, job) out, err = await cmd.shell('git apply --check ' + tmpfile, dir, job)
if out != "": if out != "":
raise ValueError(f"git apply --check failed") raise ValueError(f"git apply --check failed")
# 应用diff并处理冲突
out, err = await cmd.shell('git apply ' + tmpfile, dir, job) out, err = await cmd.shell('git apply ' + tmpfile, dir, job)
if err.startswith("error"): if err.startswith("error"):
await Log(LogType.ERROR, "The git apply operation has some conflict", job.id) await Log(LogType.ERROR, "The git apply operation has some conflict", job.id)
await cmd.shell('rm -rf ' + dir, '.', job) await cmd.shell('rm -rf ' + dir, '.', job)
return return
# 清理临时文件并将更改添加到git仓库中
await cmd.shell(f"rm -rf {tmpfile}", dir, job) await cmd.shell(f"rm -rf {tmpfile}", dir, job)
await cmd.shell('git add .', dir, job) await cmd.shell('git add .', dir, job)
#用于同步代码从一个存储库到另一个存储库处理不同类型的拉取请求例如GitHub和GitLab并在处理过程中记录相关日志。
async def sync_common(project, job, pull: PullRequestDTO): async def sync_common(project, job, pull: PullRequestDTO):
try: try:
# 记录项目信息日志
await Log(LogType.INFO, f"The project base repo is {project.base}", job.id) await Log(LogType.INFO, f"The project base repo is {project.base}", job.id)
# 记录同步代码日志
await Log(LogType.INFO, f"Sync the job code from other repo to base {project.base} repo", job.id) await Log(LogType.INFO, f"Sync the job code from other repo to base {project.base} repo", job.id)
# 创建临时目录
dir = f"/tmp/{job.project}_job_inter_{job.id}_pull_{pull.id}" dir = f"/tmp/{job.project}_job_inter_{job.id}_pull_{pull.id}"
# 记录拉取请求目录日志
await Log(LogType.INFO, f"The pull request dir is {dir}", job.id) await Log(LogType.INFO, f"The pull request dir is {dir}", job.id)
# 执行创建目录命令
await cmd.shell('mkdir ' + dir, '.', job) await cmd.shell('mkdir ' + dir, '.', job)
# 克隆Git仓库到临时目录
await cmd.shell( await cmd.shell(
f"git clone -b {job.gitlab_branch} {project.gitlab_address}", dir, job) f"git clone -b {job.gitlab_branch} {project.gitlab_address}", dir, job)
repo_dir = dir + "/" + project.name repo_dir = dir + "/" + project.name
# GitHub pull request # GitHub 拉取请求处理
if project.base != RepoType.Github and project.github_address is not None: if project.base != RepoType.Github and project.github_address is not None:
# 获取Git仓库状态
await cmd.shell('git status', repo_dir, job) await cmd.shell('git status', repo_dir, job)
new_branch = 'github_pr' + str(pull.id) new_branch = 'github_pr' + str(pull.id)
await Log(LogType.INFO, f"The new branch is {new_branch}", job.id) await Log(LogType.INFO, f"The new branch is {new_branch}", job.id)
# 创建并切换到新分支
await cmd.shell('git checkout -b ' + new_branch, repo_dir, job) await cmd.shell('git checkout -b ' + new_branch, repo_dir, job)
# 应用差异
apply_diff(project, job, pull, repo_dir) apply_diff(project, job, pull, repo_dir)
commit_msg = "Github pull request #" + str(pull.id) commit_msg = "Github pull request #" + str(pull.id)
# 提交变更
await cmd.shell(f"git commit -m \"{commit_msg}\"", repo_dir, job) await cmd.shell(f"git commit -m \"{commit_msg}\"", repo_dir, job)
# 推送新分支到远程仓库
await cmd.shell(f"git push -uv origin {new_branch}", repo_dir, job) await cmd.shell(f"git push -uv origin {new_branch}", repo_dir, job)
# 获取GitLab仓库类型
inter_type = gitlab.get_inter_repo_type(project.gitlab_address) inter_type = gitlab.get_inter_repo_type(project.gitlab_address)
if inter_type is None: if inter_type is None:
# 记录错误日志
await Log(LogType.ERROR, await Log(LogType.ERROR,
f"The {project.gitlab_address} is not belong to gitlab or antcode", job.id) f"The {project.gitlab_address} is not belong to gitlab or antcode", job.id)
else: else:
# send a merge request # 发送合并请求
if inter_type == 'gitlab': if inter_type == 'gitlab':
# Alibaba Gitlab repoCode Aone # 阿里巴巴 GitLab 仓库Code Aone
await Log(LogType.INFO, await Log(LogType.INFO,
f"Merge the pull request to internal Gitlab {project.name}", job.id) f"Merge the pull request to internal Gitlab {project.name}", job.id)
# 获取GitLab仓库ID
repo_id = gitlab.get_repo_id_from_url( repo_id = gitlab.get_repo_id_from_url(
project.gitlab_address) project.gitlab_address)
if repo_id is None: if repo_id is None:
# 记录错误日志
await Log(LogType.ERROR, await Log(LogType.ERROR,
f"We can not get the repo id {repo_id}", job.id) f"We can not get the repo id {repo_id}", job.id)
await Log(LogType.INFO, await Log(LogType.INFO,
f"The project's gitlab repo id is {repo_id}", job.id) f"The project's gitlab repo id is {repo_id}", job.id)
await Log(LogType.INFO, await Log(LogType.INFO,
f"send the merge request about the pull request #{pull.id}", job.id) f"send the merge request about the pull request #{pull.id}", job.id)
# 发送合并请求到内部GitLab
merge_to_code_aone_inter( merge_to_code_aone_inter(
repo_id, pull.id, new_branch, job.gitlab_branch) repo_id, pull.id, new_branch, job.gitlab_branch)
else: else:
# Alipay Antcode repo # 支付宝 Antcode 仓库
await Log(LogType.INFO, await Log(LogType.INFO,
f"Merge the pull request to internal Antcode {project.name}", job.id) f"Merge the pull request to internal Antcode {project.name}", job.id)
# 获取组织和项目名称
organization, name = gitlab.get_organization_and_name_from_url( organization, name = gitlab.get_organization_and_name_from_url(
project.gitlab_address) project.gitlab_address)
# 发送合并请求到内部Antcode
merge_to_antcode_inter( merge_to_antcode_inter(
job.id, pull.id, organization, name, new_branch, job.gitlab_branch) job.id, pull.id, organization, name, new_branch, job.gitlab_branch)
# update the pull request inline status # 更新拉取请求内联状态
service = PullRequestService() service = PullRequestService()
await service.update_inline_status(pull, True) await service.update_inline_status(pull, True)
await service.update_latest_commit(pull) await service.update_latest_commit(pull)
# Gitee pull request # Gitee 拉取请求处理
# TODO: Gitcode pull request # TODO: Gitcode 拉取请求处理
except: except:
# 记录错误日志
msg = f"The pull request #{pull.id} sync to the internal failed" msg = f"The pull request #{pull.id} sync to the internal failed"
await Log(LogType.ERROR, msg, job.id) await Log(LogType.ERROR, msg, job.id)
finally: finally:
# 删除临时目录
await cmd.shell('rm -rf ' + dir, '.', job) await cmd.shell('rm -rf ' + dir, '.', job)
#将拉取请求合并到GitLab指定的分支
def merge_to_code_aone_inter(repo_id: int, pull_id: int, source_branch: str, target_branch: str): def merge_to_code_aone_inter(repo_id: int, pull_id: int, source_branch: str, target_branch: str):
url = f"" url = f"" # 合并请求的URL地址
param = { param = {
'private_token': config.ACCOUNT['gitlab_token'], 'private_token': config.ACCOUNT['gitlab_token'], # GitLab访问令牌
} }
data = { data = {
"description": "Merge the pull request #" + str(pull_id), "description": "合并拉取请求 #" + str(pull_id), # 合并请求的描述
"source_branch": source_branch, "source_branch": source_branch, # 源分支
"target_branch": target_branch, "target_branch": target_branch, # 目标分支
"title": "Merge the pull request #" + str(pull_id) "title": "合并拉取请求 #" + str(pull_id) # 合并请求的标题
} }
response = requests.post(url=url, params=param, response = requests.post(url=url, params=param,
data=json.dumps(data)).json() data=json.dumps(data)).json() # 发送POST请求合并拉取请求
return response return response
#将拉取请求合并到Antcode指定的项目
async def merge_to_antcode_inter(job_id: int, pull_id: int, organization, name, source_branch, target_branch: str): async def merge_to_antcode_inter(job_id: int, pull_id: int, organization, name, source_branch, target_branch: str):
await Log(LogType.INFO, await Log(LogType.INFO,
"send the merge request about the pull request #{pull_id}", job_id) "发送有关拉取请求 #" + str(pull_id) + " 的合并请求", job_id)
headers = { headers = {
"PRIVATE-TOKEN": config.ACCOUNT['antcode_token'] "PRIVATE-TOKEN": config.ACCOUNT['antcode_token'] # Antcode访问令牌
} }
mainSiteUrl = "" mainSiteUrl = ""
organization = "oceanbase-docs" organization = "oceanbase-docs"
@ -163,33 +194,32 @@ async def merge_to_antcode_inter(job_id: int, pull_id: int, organization, name,
req_str = f"{mainSiteUrl}/api/v3/projects/find_with_namespace?path={path}" req_str = f"{mainSiteUrl}/api/v3/projects/find_with_namespace?path={path}"
response = requests.get(url=req_str, headers=headers).json() response = requests.get(url=req_str, headers=headers).json()
projectId = response['id'] projectId = response['id']
await Log(LogType.INFO, f"The Antcode project ID is {projectId}", job_id) await Log(LogType.INFO, "Antcode项目ID为" + str(projectId), job_id)
# merge request # 合并请求
merge_req_str = f"{mainSiteUrl}/api/v3/projects/{projectId}/pull_requests" merge_req_str = f"{mainSiteUrl}/api/v3/projects/{projectId}/pull_requests"
param = { param = {
'source_branch': source_branch, 'source_branch': source_branch, # 源分支
'target_branch': target_branch, 'target_branch': target_branch, # 目标分支
'squash_merge': True 'squash_merge': True
} }
response = requests.post( response = requests.post(
url=merge_req_str, param=param, headers=headers).json() url=merge_req_str, param=param, headers=headers).json()
return return
#将代码同步到OceanBase并创建任务进行处理。
async def sync_oceanbase(project, job, pull): async def sync_oceanbase(project, job, pull):
title = f"Github merge request #{pull.id} {pull.title}" title = f"Github合并请求 #{pull.id} {pull.title}" # 合并请求的标题
# Sync OceanBase code need ob flow await Log(LogType.INFO, "从Github同步代码到Inter的OceanBase", job.id)
await Log(LogType.INFO, "Sync the oceanbase code from github to inter", job.id) # 创建任务命令
# Create ob task
create_task_cmd = f"/usr/local/obdev/libexec/ob-task create --subject=\"{title}\" -T bug --branch={job.gitlab_branch} --description=\"{title}\"" create_task_cmd = f"/usr/local/obdev/libexec/ob-task create --subject=\"{title}\" -T bug --branch={job.gitlab_branch} --description=\"{title}\""
out, err = await cmd.shell(create_task_cmd, '.', job, env=dict(os.environ, AONE_ISSUE_NICK='官明')) out, err = await cmd.shell(create_task_cmd, '.', job, env=dict(os.environ, AONE_ISSUE_NICK='官明'))
issue_num = str.splitlines(out)[1].replace("[issue-id]", "").strip() issue_num = str.splitlines(out)[1].replace("[issue-id]", "").strip()
await Log(LogType.INFO, await Log(LogType.INFO,
f"The issue number is {issue_num} about the oceanbase pull request {pull.id}", job.id) f"关于OceanBase拉取请求 {pull.id} 的问题号为 {issue_num}", job.id)
task_id = issue_num.replace("T", "") task_id = issue_num.replace("T", "")
if task_id != "": if task_id != "":
await Log(LogType.INFO, await Log(LogType.INFO,
f"Create the task {task_id} successfully by ob flow", job.id) f"通过ob flow成功创建任务号 {task_id}", job.id)
await cmd.shell( await cmd.shell(
f"/usr/local/obdev/libexec/ob-flow start T{task_id} {job.gitlab_branch}", '.', job, env=dict( f"/usr/local/obdev/libexec/ob-flow start T{task_id} {job.gitlab_branch}", '.', job, env=dict(
os.environ, OB_FLOW_PROJECT='oceanbase')) os.environ, OB_FLOW_PROJECT='oceanbase'))
@ -204,12 +234,15 @@ async def sync_oceanbase(project, job, pull):
async def sync_pull_request(project, job): async def sync_pull_request(project, job):
# 从项目的GitHub地址中提取组织名和仓库名
organization, repo = github.transfer_github_to_name(project.github_address) organization, repo = github.transfer_github_to_name(project.github_address)
if organization and repo: if organization and repo:
pull_request_service = PullRequestService() pull_request_service = PullRequestService()
# 同步指定项目的拉取请求
await pull_request_service.sync_pull_request(project.name, organization, repo) await pull_request_service.sync_pull_request(project.name, organization, repo)
pull_request_service = PullRequestService() pull_request_service = PullRequestService()
# 获取数据库中与当前任务相关的拉取请求列表
pull_request_list = await pull_request_service.fetch_pull_request(project=job.project) pull_request_list = await pull_request_service.fetch_pull_request(project=job.project)
if pull_request_list and len(pull_request_list) > 0: if pull_request_list and len(pull_request_list) > 0:
@ -220,12 +253,13 @@ async def sync_pull_request(project, job):
if pull.target_branch == job.github_branch: if pull.target_branch == job.github_branch:
await Log(LogType.INFO, await Log(LogType.INFO,
f"Judge the pull request #{pull.id} of project {project.name} if need to merge", job.id) f"Judge the pull request #{pull.id} of project {project.name} if need to merge", job.id)
# 判断该拉取请求是否需要合并
need_merge = await pull_request_service.judge_pull_request_need_merge(project.name, organization, repo, pull.id) need_merge = await pull_request_service.judge_pull_request_need_merge(project.name, organization, repo, pull.id)
if need_merge: if need_merge:
await Log(LogType.INFO, await Log(LogType.INFO,
f"The pull request #{pull.id} of project {project.name} need merge", job.id) f"The pull request #{pull.id} of project {project.name} need merge", job.id)
if job.project == "oceanbase": if job.project == "oceanbase":
# Add a self config to sync the pull request what you want # 如果是OceanBase项目检查拉取请求是否在配置的ID列表中
if pull.id in config.OCEANBASE: if pull.id in config.OCEANBASE:
await sync_oceanbase(project, job, pull) await sync_oceanbase(project, job, pull)
else: else:
@ -237,7 +271,7 @@ async def sync_pull_request(project, job):
async def sync_inter_code(project, job): async def sync_inter_code(project, job):
# Judge the repo type # 判断同步类型
if job.type == SyncType.OneWay: if job.type == SyncType.OneWay:
await sync_oneway_inter_code(project, job) await sync_oneway_inter_code(project, job)
elif job.type == SyncType.TwoWay: elif job.type == SyncType.TwoWay:
@ -247,42 +281,56 @@ async def sync_inter_code(project, job):
"The job {job.github_branch}'s type of project {project.name} is wrong", job.id) "The job {job.github_branch}'s type of project {project.name} is wrong", job.id)
return return
async def sync_oneway_inter_code(project: ProjectDTO, job: JobDTO): async def sync_oneway_inter_code(project: ProjectDTO, job: JobDTO):
service = JobService() service = JobService()
# 记录同步任务开始信息到日志
await Log(LogType.INFO, "Sync the job code to outer", job.id) await Log(LogType.INFO, "Sync the job code to outer", job.id)
# 定义临时工作目录
dir = f"/data/1/tmp/{job.project}_job_outer_{job.id}" dir = f"/data/1/tmp/{job.project}_job_outer_{job.id}"
await Log(LogType.INFO, f"The sync work dir is {dir}", job.id) await Log(LogType.INFO, f"The sync work dir is {dir}", job.id)
try: try:
# 在临时目录中创建工作目录
await cmd.shell('mkdir ' + dir, '.', job) await cmd.shell('mkdir ' + dir, '.', job)
# 克隆指定分支的代码库到临时工作目录
await cmd.shell( await cmd.shell(
f"git clone -b {job.gitlab_branch} {project.gitlab_address} --depth=100", dir, job) f"git clone -b {job.gitlab_branch} {project.gitlab_address} --depth=100", dir, job)
repo_dir = dir + "/" + project.name repo_dir = dir + "/" + project.name
# 显示当前代码库状态
await cmd.shell('git status', repo_dir, job) await cmd.shell('git status', repo_dir, job)
# 添加GitHub远程仓库并获取最新更新
await cmd.shell( await cmd.shell(
f"git remote add github {project.github_address}", repo_dir, job) f"git remote add github {project.github_address}", repo_dir, job)
await cmd.shell('git fetch github', repo_dir, job) await cmd.shell('git fetch github', repo_dir, job)
# 切换到目标GitHub分支并同步更新
await cmd.shell( await cmd.shell(
f"git checkout -b out_branch github/{job.github_branch}", repo_dir, job) f"git checkout -b out_branch github/{job.github_branch}", repo_dir, job)
await cmd.shell('git checkout ' + job.gitlab_branch, repo_dir, job) await cmd.shell('git checkout ' + job.gitlab_branch, repo_dir, job)
# 如果有Gitee地址添加远程仓库并获取最新更新
if project.gitee_address: if project.gitee_address:
await cmd.shell( await cmd.shell(
f"git remote add gitee {project.gitee_address}", repo_dir, job) f"git remote add gitee {project.gitee_address}", repo_dir, job)
await cmd.shell('git fetch gitee', repo_dir, job) await cmd.shell('git fetch gitee', repo_dir, job)
# 如果有Code China地址添加远程仓库并获取最新更新
if project.code_china_address: if project.code_china_address:
await cmd.shell( await cmd.shell(
f"git remote add csdn {project.code_china_address}", repo_dir, job) f"git remote add csdn {project.code_china_address}", repo_dir, job)
result, err = await cmd.shell('git status', repo_dir, job) result, err = await cmd.shell('git status', repo_dir, job)
# fetch the latest commit # 获取最新提交信息
latestCommit = await service.get_job_lateset_commit(job.id) latestCommit = await service.get_job_lateset_commit(job.id)
await Log(LogType.INFO, 'The lastest commit is ' + latestCommit, job.id) await Log(LogType.INFO, 'The lastest commit is ' + latestCommit, job.id)
if latestCommit == 'no_commit': if latestCommit == 'no_commit':
# 如果没有最新提交,则获取最新合并提交
result, err = await cmd.shell( result, err = await cmd.shell(
f"git log HEAD^1..HEAD --oneline --merges", repo_dir, job) f"git log HEAD^1..HEAD --oneline --merges", repo_dir, job)
commit = result.split(" ")[0] commit = result.split(" ")[0]
@ -290,6 +338,7 @@ async def sync_oneway_inter_code(project: ProjectDTO, job: JobDTO):
await patch_every_commit(repo_dir, project, job, commit) await patch_every_commit(repo_dir, project, job, commit)
return return
else: else:
# 否则获取从最新提交到HEAD的所有合并提交
result, err = await cmd.shell( result, err = await cmd.shell(
"git log "+latestCommit + "..HEAD --oneline --merges", repo_dir, job) "git log "+latestCommit + "..HEAD --oneline --merges", repo_dir, job)
@ -297,6 +346,7 @@ async def sync_oneway_inter_code(project: ProjectDTO, job: JobDTO):
await Log(LogType.INFO, await Log(LogType.INFO,
f"The commit {latestCommit} is the newest commit on the remote branch", job.id) f"The commit {latestCommit} is the newest commit on the remote branch", job.id)
else: else:
# 反向遍历所有提交,并逐一进行代码补丁操作
commit_info_list = str.splitlines(result) commit_info_list = str.splitlines(result)
commit_info_list.reverse() commit_info_list.reverse()
for commit_info in commit_info_list: for commit_info in commit_info_list:
@ -304,152 +354,153 @@ async def sync_oneway_inter_code(project: ProjectDTO, job: JobDTO):
await Log(LogType.INFO, "patch the commit " + commit, job.id) await Log(LogType.INFO, "patch the commit " + commit, job.id)
await patch_every_commit(repo_dir, project, job, commit) await patch_every_commit(repo_dir, project, job, commit)
except: except:
# 捕获异常情况,并记录错误信息到日志
msg = f"Sync the code from inter to outer of project {project.name} branch {job.github_branch} failed" msg = f"Sync the code from inter to outer of project {project.name} branch {job.github_branch} failed"
await Log(LogType.ERROR, msg, job.id) await Log(LogType.ERROR, msg, job.id)
finally: finally:
# 最终清理临时工作目录
await cmd.shell(f"rm -rf {dir}", '.', job) await cmd.shell(f"rm -rf {dir}", '.', job)
await Log(LogType.INFO, f"remove the temper repo folder {dir}", job.id) await Log(LogType.INFO, f"remove the temper repo folder {dir}", job.id)
return return
#该异步函数用于同步项目代码库的内容到多个远程仓库包括GitHub、Gitee和Code China并更新最新的提交哈希值。
async def sync_twoway_inter_code(project, job): async def sync_twoway_inter_code(project, job):
service = JobService() service = JobService()
await Log(LogType.INFO, "Sync the job document to outer", job.id) await Log(LogType.INFO, "Sync the job document to outer", job.id) # 记录同步任务开始信息到日志
dir = "/tmp/" + job.project+"_job_outer_"+str(job.id) dir = "/tmp/" + job.project+"_job_outer_"+str(job.id) # 定义临时工作目录
await Log(LogType.INFO, f"The sync work dir is {dir}", job.id) await Log(LogType.INFO, f"The sync work dir is {dir}", job.id) # 记录临时工作目录路径到日志
try: try:
await cmd.shell('mkdir ' + dir, '.', job) await cmd.shell('mkdir ' + dir, '.', job) # 创建临时工作目录
await cmd.shell( await cmd.shell(
f"git clone -b {job.gitlab_branch} {project.gitlab_address}", dir, job) f"git clone -b {job.gitlab_branch} {project.gitlab_address}", dir, job) # 克隆指定分支的代码库到临时工作目录
repo_dir = dir + "/" + project.name repo_dir = dir + "/" + project.name # 设置代码库目录路径
await cmd.shell('git status', repo_dir, job) await cmd.shell('git status', repo_dir, job) # 显示当前代码库状态
await cmd.shell('git remote add github ' + await cmd.shell('git remote add github ' +
project.github_address, repo_dir, job) project.github_address, repo_dir, job) # 添加GitHub远程仓库地址
await cmd.shell('git fetch github', repo_dir, job) await cmd.shell('git fetch github', repo_dir, job) # 获取GitHub远程仓库的更新
await cmd.shell('git pull -r github ' + job.github_branch, repo_dir, job) await cmd.shell('git pull -r github ' + job.github_branch, repo_dir, job) # 从GitHub远程仓库拉取并重新基于合并
await cmd.shell( await cmd.shell(
f"git push origin {job.gitlab_branch} -f", repo_dir, job) f"git push origin {job.gitlab_branch} -f", repo_dir, job) # 强制推送更新到GitLab远程仓库
await cmd.shell( await cmd.shell(
f"git push github {job.github_branch} -f", repo_dir, job) f"git push github {job.github_branch} -f", repo_dir, job) # 强制推送更新到GitHub远程仓库
if project.gitee_address: if project.gitee_address:
await cmd.shell('git remote add gitee ' + await cmd.shell('git remote add gitee ' +
project.gitee_address, repo_dir, job) project.gitee_address, repo_dir, job) # 如果有Gitee地址添加Gitee远程仓库地址
await cmd.shell('git fetch gitee', repo_dir, job) await cmd.shell('git fetch gitee', repo_dir, job) # 获取Gitee远程仓库的更新
await cmd.shell( await cmd.shell(
f"git push gitee {job.gitlab_branch}:{job.gitee_branch}", repo_dir, job) f"git push gitee {job.gitlab_branch}:{job.gitee_branch}", repo_dir, job) # 推送更新到Gitee远程仓库
if project.code_china_address: if project.code_china_address:
await cmd.shell('git remote add csdn ' + await cmd.shell('git remote add csdn ' +
project.code_china_address, repo_dir, job) project.code_china_address, repo_dir, job) # 如果有Code China地址添加Code China远程仓库地址
result, err = await cmd.shell('git status', repo_dir, job) result, err = await cmd.shell('git status', repo_dir, job) # 显示当前代码库状态
await cmd.shell( await cmd.shell(
f"git push csdn {job.gitlab_branch}:{job.code_china_branch}", repo_dir, job) f"git push csdn {job.gitlab_branch}:{job.code_china_branch}", repo_dir, job) # 推送更新到Code China远程仓库
# update the latest commit hash # 更新最新提交的哈希值
result, err = await cmd.shell( result, err = await cmd.shell(
"git log HEAD~1..HEAD --oneline", repo_dir, job) "git log HEAD~1..HEAD --oneline", repo_dir, job) # 获取最新提交的信息
commit = result.split(" ")[0] commit = result.split(" ")[0] # 提取最新提交的哈希值
await service.update_job_lateset_commit(job.id, commit) await service.update_job_lateset_commit(job.id, commit) # 更新任务中的最新提交哈希值
except: except:
msg = f"Sync the document from inter to outer of project {project.name} branch {job.github_branch} failed" msg = f"Sync the document from inter to outer of project {project.name} branch {job.github_branch} failed"
await Log(LogType.Error, msg, job.id) await Log(LogType.Error, msg, job.id) # 捕获异常情况,并记录错误信息到日志
finally: finally:
await cmd.shell(f"rm -rf {dir}", '.', job) await cmd.shell(f"rm -rf {dir}", '.', job) # 最终清理临时工作目录
await Log(LogType.INFO, f"remove the temper repo folder {dir}", job.id) await Log(LogType.INFO, f"remove the temper repo folder {dir}", job.id) # 记录删除临时工作目录的日志
return return
#该函数用于从给定的作者内容字符串中提取作者的邮箱地址,返回邮箱地址的用户名部分。
def get_author_from_oceanbase(author_content: str) -> Optional[str]: def get_author_from_oceanbase(author_content: str) -> Optional[str]:
partten = r'Author : (.*) \((.*)\)' partten = r'Author : (.*) \((.*)\)' # 定义正则表达式模式,用于匹配作者信息
matchObj = re.match(partten, author_content, re.M | re.I) matchObj = re.match(partten, author_content, re.M | re.I) # 使用正则表达式匹配作者内容
if matchObj: if matchObj:
author = matchObj.group(2) author = matchObj.group(2) # 提取匹配到的第二部分,即邮箱地址
return author.split('#')[0] return author.split('#')[0] # 返回邮箱地址的用户名部分
return None return None # 如果没有匹配到返回None
#函数负责根据给定的commit执行一系列git操作包括切换分支、重置提交、获取提交信息、处理补丁文件等。最终目的是将代码从内部合并到外部并处理可能的冲突。
async def patch_every_commit(dir, project, job, commit): async def patch_every_commit(dir, project, job, commit):
service = JobService() service = JobService() # 创建一个JobService实例
try: try:
await cmd.shell('git status', dir, job) await cmd.shell('git status', dir, job) # 在指定目录执行'git status'命令检查当前git状态
await cmd.shell('git checkout ' + job.gitlab_branch, dir, job) await cmd.shell('git checkout ' + job.gitlab_branch, dir, job) # 切换到job对应的gitlab分支
await cmd.shell('git pull -r origin ' + job.gitlab_branch, dir, job) await cmd.shell('git pull -r origin ' + job.gitlab_branch, dir, job) # 从远程仓库拉取最新代码并且rebase到当前分支上
await cmd.shell('git reset --hard ' + commit, dir, job) await cmd.shell('git reset --hard ' + commit, dir, job) # 重置当前分支到指定commit
# Get the commit comment # 获取此次提交的日志信息
output, err = await cmd.shell("git log -1", dir, job) output, err = await cmd.shell("git log -1", dir, job)
email, err = await cmd.shell("git log --format='%ae' -1", dir, job) email, err = await cmd.shell("git log --format='%ae' -1", dir, job) # 获取此次提交的作者email
if email is None: if email is None:
raise ValueError("The commit has no email") raise ValueError("The commit has no email") # 如果没有email抛出异常
await Log(LogType.INFO, f"The commit {commit} email is {email}", job.id) await Log(LogType.INFO, f"The commit {commit} email is {email}", job.id) # 记录提交的email信息
if project.name == "oceanbase": if project.name == "oceanbase":
author_string = str.splitlines(output)[8].strip() author_string = str.splitlines(output)[8].strip() # 获取第9行作者字符串
await Log(LogType.INFO, await Log(LogType.INFO,
f"The author string is {author_string}", job.id) f"The author string is {author_string}", job.id) # 记录作者字符串信息
domain = get_author_from_oceanbase(author_string) domain = get_author_from_oceanbase(author_string) # 从oceanbase项目获取作者域
else: else:
domain = author.get_author_domain(email) domain = author.get_author_domain(email) # 通过email获取作者域
if domain is None: if domain is None:
raise ValueError("The commit author has no ali domain") raise ValueError("The commit author has no ali domain") # 如果没有作者域,抛出异常
await Log(LogType.INFO, f"The commit author ali domain is {domain}", job.id) await Log(LogType.INFO, f"The commit author ali domain is {domain}", job.id) # 记录作者域信息
content = str.splitlines(output)[5].strip() content = str.splitlines(output)[5].strip() # 获取第6行的提交内容
await Log(LogType.INFO, f"content is {content}", job.id) await Log(LogType.INFO, f"content is {content}", job.id) # 记录提交内容信息
if content is None or content == "": if content is None or content == "":
raise ValueError("The commit has no commit content") raise ValueError("The commit has no commit content") # 如果没有提交内容,抛出异常
await Log(LogType.INFO, f"The commit {commit} content is {content}", job.id) await Log(LogType.INFO, f"The commit {commit} content is {content}", job.id) # 记录提交内容信息
# TODO if find the commit is from github, merge the pull request # TODO 如果发现提交来自github合并pull request
if content.startswith("Github Merge"): if content.startswith("Github Merge"):
pr_id = int(content.split()[3].replace('#', '')) pr_id = int(content.split()[3].replace('#', '')) # 提取PR id
pr_service = PullRequestService() pr_service = PullRequestService() # 创建PullRequestService实例
organization, repo = github.transfer_github_to_name( organization, repo = github.transfer_github_to_name(
project.github_address) project.github_address) # 获取组织和仓库名称
ans = await pr_service.merge_pull_request_code(organization, repo, pr_id) ans = await pr_service.merge_pull_request_code(organization, repo, pr_id) # 合并PR
if ans is None: if ans is None:
return return
# if the repo has .ce file, it means we should do something before merge # 如果仓库中包含.ce文件则在合并前执行相关操作
# the code from inter to outer
ce_file = Path(dir + '/.ce') ce_file = Path(dir + '/.ce')
if ce_file.is_file(): if ce_file.is_file():
await cmd.shell('bash .ce', dir, job) await cmd.shell('bash .ce', dir, job) # 执行.ce文件中的脚本
else: else:
await Log(LogType.INFO, await Log(LogType.INFO,
f"There is no .ce file in the project {project.name}", job.id) f"There is no .ce file in the project {project.name}", job.id) # 记录没有.ce文件的日志信息
# TODO check git diff apply --check # TODO 检查git diff apply --check
diff, err = await cmd.shell("git diff out_branch", dir, job) diff, err = await cmd.shell("git diff out_branch", dir, job) # 获取与目标分支的差异
if diff == "": if diff == "":
# The diff is empty, save the commit and return # 如果没有差异,保存提交并返回
await cmd.shell('git reset --hard', dir, job) await cmd.shell('git reset --hard', dir, job) # 重置当前分支到最新提交
await service.update_job_lateset_commit(job.id, commit) await service.update_job_lateset_commit(job.id, commit) # 更新job的最新提交信息
return return
patch_file = '/tmp/' + job.github_branch + '_patch' patch_file = '/tmp/' + job.github_branch + '_patch' # 创建补丁文件路径
await cmd.shell('rm -rf ' + patch_file, dir, job) await cmd.shell('rm -rf ' + patch_file, dir, job) # 删除可能存在的旧补丁文件
with open(patch_file, "w") as diff_file: with open(patch_file, "w") as diff_file: # 打开补丁文件用于写入
diff, err = await cmd.shell("git diff out_branch", dir, job) diff, err = await cmd.shell("git diff out_branch", dir, job) # 再次获取差异
diff_file.write(diff) diff_file.write(diff) # 写入补丁文件
await cmd.shell('git reset --hard', dir, job) await cmd.shell('git reset --hard', dir, job) # 重置当前分支到最新提交
await cmd.shell('git checkout out_branch', dir, job) await cmd.shell('git checkout out_branch', dir, job) # 切换到目标分支
# git apply --check first # git apply --check first
# out, err = await cmd.shell('git apply --check ' + patch_file, dir, job) # out, err = await cmd.shell('git apply --check ' + patch_file, dir, job)
if err != "": if err != "":
raise ValueError( raise ValueError(
f"The commit {commit} has conflict to the branch {job.github_branch}") f"The commit {commit} has conflict to the branch {job.github_branch}") # 如果有冲突,抛出异常
await cmd.shell('git apply ' + patch_file, dir, job) await cmd.shell('git apply ' + patch_file, dir, job) # 应用补丁文件
await cmd.shell('git add .', dir, job) await cmd.shell('git add .', dir, job) # 添加所有更改到暂存区
await cmd.shell(f"git commit -m \"{content}\"", dir, job) await cmd.shell(f"git commit -m \"{content}\"", dir, job) # 提交更改
# TODO:change commit author # TODO:change commit author
out = await author.get_github_author_and_email(domain) out = await author.get_github_author_and_email(domain)