同步日志记录方式调整

sql/update_20240428.sql

@@ -0,0 +1,3 @@
+
+ALTER TABLE `repo_sync_log`
+MODIFY COLUMN `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT;

src/api/Sync_config.py

@@ -257,10 +257,19 @@ class SyncDirection(Controller):
     async def get_logs(
             self, request: Request, user: str = Depends(user),
             repo_name: str = Path(..., description="仓库名称"),
-            branch_id: int = Query(None, description="分支id（仓库粒度无需输入）")
+            branch_id: int = Query(None, description="分支id（仓库粒度无需输入）"),
+            page_num: int = Query(1, description="页数"), page_size: int = Query(10, description="条数"),
+            create_sort: bool = Query(False, description="创建时间排序， 默认倒序")
     ):
         api_log(LogType.INFO, f"用户 {user} 使用 GET 方法访问接口 {request.url.path} ", user)
-        data = await self.log_service.get_logs(repo_name=repo_name, branch_id=branch_id)
+        data = await self.log_service.get_logs(repo_name=repo_name, branch_id=branch_id,
+                                               page_num=page_num, page_size=page_size, create_sort=create_sort)
+        if not data:
+            return SYNCResponse(
+                code_status=Status.CHECK_IN.code,
+                data=data,
+                msg=Status.CHECK_IN.msg
+            )
         return SYNCResponse(
             code_status=Status.SUCCESS.code,
             data=data,

src/base/status_code.py

@@ -24,6 +24,7 @@ class Status(Enum):
     BRANCH_DELETE = (10012, "仓库中不存在此分支")
     NOT_DATA = (10013, "没有数据")
     GRANULARITY = (10014, "仓库粒度同步，没有分支信息")
+    CHECK_IN = (10015, "请检查输入的仓库和分支ID信息是否对应")
     # git执行异常
     PERMISSION_DENIED = (20001, "SSH 密钥未授权或未添加")
     REPO_NOT_FOUND = (20002, "仓库不存在或私有仓库访问权限不足")

src/dao/sync_config.py

@@ -3,7 +3,7 @@ from sqlalchemy.exc import NoResultFound
 from src.do.sync_config import SyncBranchMapping, SyncRepoMapping, LogDO
 from .mysql_ao import MysqlAO
 from src.utils.base import Singleton
-from src.dto.sync_config import AllRepoDTO, GetBranchDTO, SyncRepoDTO, SyncBranchDTO, RepoDTO, BranchDTO
+from src.dto.sync_config import AllRepoDTO, GetBranchDTO, SyncRepoDTO, SyncBranchDTO, RepoDTO, BranchDTO, LogDTO
 from typing import List
 from src.do.sync_config import SyncDirect, SyncType
 
@@ -231,6 +231,14 @@ class LogDAO(BaseDAO, metaclass=Singleton):
                 session.add(do)
             await session.commit()
 
+    async def insert_sync_repo_log(self, repo_name, direct, log_content, first_time, last_time):
+        async with self._async_session() as session:
+            async with session.begin():
+                do = LogDO(repo_name=repo_name, sync_direct=direct, log=log_content,
+                           created_at=first_time, update_at=last_time)
+                session.add(do)
+            await session.commit()
+
     async def update_sync_repo_log(self, repo_name, direct, log_content):
         async with self._async_session() as session:
             async with session.begin():
@@ -253,6 +261,14 @@ class LogDAO(BaseDAO, metaclass=Singleton):
                 session.add(do)
             await session.commit()
 
+    async def insert_branch_log(self, repo_name, direct, branch_id, commit_id, log_content, first_time, last_time):
+        async with self._async_session() as session:
+            async with session.begin():
+                do = LogDO(repo_name=repo_name, sync_direct=direct, branch_id=branch_id,
+                           commit_id=commit_id, log=log_content, created_at=first_time, update_at=last_time)
+                session.add(do)
+            await session.commit()
+
     async def update_branch_log(self, repo_name, direct, branch_id, commit_id, log_content):
         async with self._async_session() as session:
             async with session.begin():
@@ -267,3 +283,24 @@ class LogDAO(BaseDAO, metaclass=Singleton):
                 await session.execute(stmt)
             await session.commit()
 
+    async def get_log(self, repo_name: str, branch_id: int, page_number: int, page_size: int, create_sort: bool) -> List[LogDTO]:
+        async with self._async_session() as session:
+            async with session.begin():
+                stmt = select(LogDO).where(LogDO.repo_name == repo_name, LogDO.branch_id == branch_id)
+                create_order = LogDO.created_at if create_sort else LogDO.created_at.desc()
+                stmt = stmt.order_by(create_order).offset((page_number - 1) * page_size).limit(page_size)
+                do_list: List[LogDO] = (await session.execute(stmt)).scalars().all()
+                datas = []
+                for do in do_list:
+                    data = LogDTO(
+                        id=do.id,
+                        branch_id=do.branch_id,
+                        repo_name=do.repo_name,
+                        commit_id=do.commit_id,
+                        sync_direct=do.sync_direct.name,
+                        log=str(do.log),
+                        created_at=str(do.created_at),
+                        update_at=str(do.update_at)
+                    )
+                    datas.append(data)
+                return datas

src/do/sync_config.py

@@ -56,7 +56,7 @@ class LogDO(DataObject):
     sync_direct = Column(Enum(SyncDirect), comment="同步方向")
     log = Column(Text, comment="同步日志")
     # log_history = Column(Text, comment="历史日志")
-    created_at = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP'), comment="创建时间")
-    update_at = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP'), comment="更新时间")
+    created_at = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP'), comment="同步执行时间")
+    update_at = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP'), comment="同步结束时间")
 
 

src/service/sync_config.py

@@ -2,6 +2,7 @@ import re
 from typing import List, Union, Optional, Dict
 from .service import Service
 from src.utils import base
+from datetime import datetime
 from src.dao.sync_config import SyncBranchDAO, SyncRepoDAO, LogDAO
 from src.dto.sync_config import SyncBranchDTO, SyncRepoDTO, RepoDTO, AllRepoDTO, GetBranchDTO, LogDTO, BranchDTO, ModifyRepoDTO
 from src.do.sync_config import SyncDirect, SyncType
@@ -166,12 +167,18 @@ class LogService(Service):
         log_history = f"{log_path}/sync_{repo_name}_history.log"
         with open(log_history, 'a') as log_:
             log_.write(log_content)
-        stm = {"repo_name": repo_name, "branch_id": None, "commit_id": None}
-        log = await self.sync_log_dao.filter(**stm)
-        if len(log) < 1:
-            await self.sync_log_dao.init_sync_repo_log(repo_name=repo_name, direct=direct, log_content=log_content)
-        else:
-            await self.sync_log_dao.update_sync_repo_log(repo_name=repo_name, direct=direct, log_content=log_content)
+        # 使用正则表达式匹配所有的日期和时间格式
+        timestamps = re.findall(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', log_content)
+        first_time = timestamps[0] if timestamps else datetime.now()
+        last_time = timestamps[-1] if timestamps else datetime.now()
+        await self.sync_log_dao.insert_sync_repo_log(repo_name=repo_name, direct=direct, log_content=log_content,
+                                                     first_time=first_time, last_time=last_time)
+        # stm = {"repo_name": repo_name, "branch_id": None, "commit_id": None}
+        # log = await self.sync_log_dao.filter(**stm)
+        # if len(log) < 1:
+        #     await self.sync_log_dao.init_sync_repo_log(repo_name=repo_name, direct=direct, log_content=log_content)
+        # else:
+        #     await self.sync_log_dao.update_sync_repo_log(repo_name=repo_name, direct=direct, log_content=log_content)
 
     async def insert_branch_log(self, repo_name: str, direct: str, branch_id: int, commit_id: str):
         addr = f"{log_path}/sync_{repo_name}_{branch_id}.log"
@@ -180,28 +187,20 @@ class LogService(Service):
         log_history = f"{log_path}/sync_{repo_name}_{branch_id}_history.log"
         with open(log_history, 'a') as log_:
             log_.write(log_content)
-        stm = {"repo_name": repo_name, "branch_id": branch_id}
-        log = await self.sync_log_dao.filter(**stm)
-        if len(log) < 1:
-            await self.sync_log_dao.init_branch_log(repo_name, direct, branch_id, commit_id, log_content)
-        else:
-            await self.sync_log_dao.update_branch_log(repo_name, direct, branch_id, commit_id, log_content)
+        # 使用正则表达式匹配所有的日期和时间格式
+        timestamps = re.findall(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', log_content)
+        first_time = timestamps[0] if timestamps else datetime.now()
+        last_time = timestamps[-1] if timestamps else datetime.now()
+        await self.sync_log_dao.insert_branch_log(repo_name, direct, branch_id, commit_id,
+                                                  log_content, first_time, last_time)
+        # stm = {"repo_name": repo_name, "branch_id": branch_id}
+        # log = await self.sync_log_dao.filter(**stm)
+        # if len(log) < 1:
+        #     await self.sync_log_dao.init_branch_log(repo_name, direct, branch_id, commit_id, log_content)
+        # else:
+        #     await self.sync_log_dao.update_branch_log(repo_name, direct, branch_id, commit_id, log_content)
 
-    async def get_logs(self, repo_name: str, branch_id: int) -> Optional[List[LogDTO]]:
-        stm = {"repo_name": repo_name, "branch_id": branch_id}
-        logs_repo = await self.sync_log_dao.filter(**stm)
-        datas = []
-        for do in logs_repo:
-            data = LogDTO(
-                id=do.id,
-                branch_id=do.branch_id,
-                repo_name=do.repo_name,
-                commit_id=do.commit_id,
-                sync_direct=do.sync_direct.name,
-                log=str(do.log),
-                # log_history=str(do.log_history),
-                created_at=str(do.created_at),
-                update_at=str(do.update_at)
-            )
-            datas.append(data)
-        return datas
+    async def get_logs(self, repo_name: str, branch_id: int, page_num: int, page_size: int, create_sort: bool) -> Optional[List[LogDTO]]:
+        logs = await self.sync_log_dao.get_log(repo_name=repo_name, branch_id=branch_id,
+                                               page_number=page_num, page_size=page_size, create_sort=create_sort)
+        return logs