删除无用文件
This commit is contained in:
parent
6d8ab9c802
commit
3794e4274e
|
|
@ -1,317 +0,0 @@
|
||||||
import sys
|
|
||||||
|
|
||||||
sys.path.append("..") # 添加util所在目录
|
|
||||||
|
|
||||||
import time
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import argparse
|
|
||||||
from pathlib import Path
|
|
||||||
from pymongo import MongoClient
|
|
||||||
from util import logging_init
|
|
||||||
|
|
||||||
|
|
||||||
DB = "JiraRepos"
|
|
||||||
|
|
||||||
REPOS = [
|
|
||||||
"Apache",
|
|
||||||
"Hyperledger",
|
|
||||||
"IntelDAOS",
|
|
||||||
"JFrog",
|
|
||||||
"Jira",
|
|
||||||
"JiraEcosystem",
|
|
||||||
"MariaDB",
|
|
||||||
"Mindville",
|
|
||||||
"Mojang",
|
|
||||||
"MongoDB",
|
|
||||||
"Qt",
|
|
||||||
"RedHat",
|
|
||||||
"Sakai",
|
|
||||||
"SecondLife",
|
|
||||||
"Sonatype",
|
|
||||||
"Spring",
|
|
||||||
]
|
|
||||||
|
|
||||||
EPIC_FIELD = {
|
|
||||||
"Apache": "customfield_12311120",
|
|
||||||
"Hyperledger": "customfield_10006",
|
|
||||||
"IntelDAOS": "customfield_10092",
|
|
||||||
"JFrog": "customfield_10806",
|
|
||||||
"Jira": "customfield_12931",
|
|
||||||
"JiraEcosystem": "customfield_12180",
|
|
||||||
"MariaDB": "customfield_10600",
|
|
||||||
"MindVille": "customfield_10000",
|
|
||||||
"MongoDB": "customfield_10857",
|
|
||||||
"Qt": "customfield_10400",
|
|
||||||
"Redhat": "customfield_12311140",
|
|
||||||
"Sakai": "customfield_10772",
|
|
||||||
"SecondLife": "customfield_10871",
|
|
||||||
"Sonatype": "customfield_11500",
|
|
||||||
"Spring": "customfield_10680",
|
|
||||||
}
|
|
||||||
|
|
||||||
ISSUE_DIR = Path("../../data/raw/issues")
|
|
||||||
ISSUE_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
LINK_DIR = Path("../../data/raw/links")
|
|
||||||
LINK_DIR.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
|
|
||||||
def extract_issues(db, repo: str):
|
|
||||||
"""提取Jira issues数据"""
|
|
||||||
|
|
||||||
collection = db[repo]
|
|
||||||
cursor = collection.find({}) # 查询所有document
|
|
||||||
|
|
||||||
issues = []
|
|
||||||
issue_ct, comment_ct = 0, 0 # issue和comment总数
|
|
||||||
|
|
||||||
for document in cursor:
|
|
||||||
try:
|
|
||||||
issue_key = document["key"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
issuetype = document["fields"]["issuetype"]["name"]
|
|
||||||
except Exception:
|
|
||||||
issuetype = "None"
|
|
||||||
|
|
||||||
try:
|
|
||||||
status = document["fields"]["status"]["name"] # 状态
|
|
||||||
except Exception:
|
|
||||||
status = "None"
|
|
||||||
|
|
||||||
try:
|
|
||||||
priority = document["fields"]["priority"]["name"] # 优先级
|
|
||||||
except Exception:
|
|
||||||
priority = "None"
|
|
||||||
|
|
||||||
project = document["fields"]["project"]["name"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
resolution = document["fields"]["resolution"]["name"] # 解决与否
|
|
||||||
except Exception:
|
|
||||||
resolution = "Open"
|
|
||||||
|
|
||||||
try:
|
|
||||||
component_arr = document["fields"]["components"]
|
|
||||||
components = []
|
|
||||||
for item in component_arr:
|
|
||||||
components.append(item["name"])
|
|
||||||
except Exception:
|
|
||||||
components = []
|
|
||||||
|
|
||||||
try:
|
|
||||||
created = document["fields"]["created"]
|
|
||||||
except Exception:
|
|
||||||
created = "None"
|
|
||||||
|
|
||||||
try:
|
|
||||||
updated = document["fields"]["updated"]
|
|
||||||
except Exception:
|
|
||||||
updated = "None"
|
|
||||||
|
|
||||||
try:
|
|
||||||
summary = document["fields"]["summary"]
|
|
||||||
except Exception:
|
|
||||||
summary = " "
|
|
||||||
|
|
||||||
try:
|
|
||||||
description = document["fields"]["description"]
|
|
||||||
except Exception:
|
|
||||||
description = " "
|
|
||||||
|
|
||||||
try:
|
|
||||||
comment_arr = document["fields"]["comments"]
|
|
||||||
comments = []
|
|
||||||
for item in comment_arr:
|
|
||||||
comments.append(item["body"])
|
|
||||||
comment_ct += 1
|
|
||||||
except:
|
|
||||||
comments = []
|
|
||||||
|
|
||||||
issue = {
|
|
||||||
"issue_key": issue_key,
|
|
||||||
"type": issuetype,
|
|
||||||
"status": status,
|
|
||||||
"priority": priority,
|
|
||||||
"resolution": resolution,
|
|
||||||
"project": project,
|
|
||||||
"created": created,
|
|
||||||
"updated": updated,
|
|
||||||
"title": summary,
|
|
||||||
"description": description,
|
|
||||||
"comments": comments,
|
|
||||||
"components": components,
|
|
||||||
}
|
|
||||||
|
|
||||||
issues.append(issue)
|
|
||||||
issue_ct += 1
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
file_name = ISSUE_DIR / (repo + ".json")
|
|
||||||
with open(file_name, "w", errors="surrogatepass", encoding="utf-8") as f:
|
|
||||||
json.dump(issues, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
logging.info(
|
|
||||||
f"Extracted {issue_ct} issues, {comment_ct} comments from {repo} repo."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def extract_links(db, repo: str):
|
|
||||||
"""提取Jira issue links数据"""
|
|
||||||
|
|
||||||
collection = db[repo]
|
|
||||||
cursor = collection.find({}) # 查询所有document
|
|
||||||
|
|
||||||
links = []
|
|
||||||
|
|
||||||
for document in cursor:
|
|
||||||
try:
|
|
||||||
issue_key = document["key"]
|
|
||||||
|
|
||||||
issuelinks = document["fields"]["issuelinks"]
|
|
||||||
|
|
||||||
# 保存通用类型links
|
|
||||||
for issue_link in issuelinks:
|
|
||||||
type = issue_link["type"]["name"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
in_issue_key = issue_key
|
|
||||||
out_issue_key = issue_link["outwardIssue"]["key"]
|
|
||||||
except Exception:
|
|
||||||
out_issue_key = issue_key
|
|
||||||
in_issue_key = issue_link["inwardIssue"]["key"]
|
|
||||||
|
|
||||||
link_key = in_issue_key + "_" + out_issue_key
|
|
||||||
|
|
||||||
link = {
|
|
||||||
"link_key": link_key,
|
|
||||||
"type": type,
|
|
||||||
"in_issue_key": in_issue_key,
|
|
||||||
"out_issue_key": out_issue_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
links.append(link)
|
|
||||||
|
|
||||||
# 单独处理Subtask类型links
|
|
||||||
subtasks = document["fields"]["subtasks"]
|
|
||||||
for subtask in subtasks:
|
|
||||||
type = "Subtask"
|
|
||||||
in_issue_key = issue_key
|
|
||||||
out_issue_key = subtask["key"]
|
|
||||||
|
|
||||||
link_key = in_issue_key + "_" + out_issue_key
|
|
||||||
|
|
||||||
link = {
|
|
||||||
"link_key": link_key,
|
|
||||||
"type": type,
|
|
||||||
"in_issue_key": in_issue_key,
|
|
||||||
"out_issue_key": out_issue_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
links.append(link)
|
|
||||||
|
|
||||||
# 单独处理Epic类型links
|
|
||||||
try:
|
|
||||||
epic = document["fields"][EPIC_FIELD[repo]]
|
|
||||||
type = "Epic-Relation"
|
|
||||||
in_issue_key = issue_key
|
|
||||||
out_issue_key = epic
|
|
||||||
|
|
||||||
link_key = in_issue_key + "_" + out_issue_key
|
|
||||||
|
|
||||||
link = {
|
|
||||||
"link_key": link_key,
|
|
||||||
"type": type,
|
|
||||||
"in_issue_key": in_issue_key,
|
|
||||||
"out_issue_key": out_issue_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
links.append(link)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 单独处理RedHat Repo中的特殊类型links
|
|
||||||
if repo == "RedHat":
|
|
||||||
try:
|
|
||||||
parent = document["fields"]["customfield_12313140"]
|
|
||||||
type = "Parent-Relation"
|
|
||||||
in_issue_key = issue_key
|
|
||||||
out_issue_key = parent
|
|
||||||
|
|
||||||
link_key = in_issue_key + "_" + out_issue_key
|
|
||||||
|
|
||||||
link = {
|
|
||||||
"link_key": link_key,
|
|
||||||
"type": type,
|
|
||||||
"in_issue_key": in_issue_key,
|
|
||||||
"out_issue_key": out_issue_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
links.append(link)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
feature = document["fields"]["customfield_12318341"]
|
|
||||||
type = "Feature-Relation"
|
|
||||||
in_issue_key = issue_key
|
|
||||||
out_issue_key = feature
|
|
||||||
|
|
||||||
link_key = in_issue_key + "_" + out_issue_key
|
|
||||||
|
|
||||||
link = {
|
|
||||||
"link_key": link_key,
|
|
||||||
"type": type,
|
|
||||||
"in_issue_key": in_issue_key,
|
|
||||||
"out_issue_key": out_issue_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
links.append(link)
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
file_name = LINK_DIR / (repo + ".json")
|
|
||||||
with open(file_name, "w", errors="surrogatepass", encoding="utf-8") as f:
|
|
||||||
json.dump(links, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
logging.info(f"Extracted links from {repo} repo. done!")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
# 解析命令行参数
|
|
||||||
parser = argparse.ArgumentParser(description="Extract Jira issues data")
|
|
||||||
parser.add_argument("--host", default="localhost")
|
|
||||||
parser.add_argument("--port", type=int, default=27017)
|
|
||||||
parser.add_argument("--username", default=None)
|
|
||||||
parser.add_argument("--password", default=None)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
logging_init(log_filename="extract_jira_data", log_dir="../../log/preprocess")
|
|
||||||
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
# 创建连接，访问MongoDB
|
|
||||||
with MongoClient(
|
|
||||||
host=args.host,
|
|
||||||
port=args.port,
|
|
||||||
username=args.username,
|
|
||||||
password=args.password,
|
|
||||||
serverSelectionTimeoutMS=5000,
|
|
||||||
) as client:
|
|
||||||
db = client[DB]
|
|
||||||
|
|
||||||
for repo in REPOS:
|
|
||||||
extract_issues(db, repo)
|
|
||||||
extract_links(db, repo)
|
|
||||||
logging.info("=" * 20)
|
|
||||||
|
|
||||||
end_time = time.perf_counter()
|
|
||||||
|
|
||||||
logging.info(f"Time cost: {end_time - start_time:.3f}s")
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
from .log_helper import logging_init
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
import sys
|
|
||||||
import logging
|
|
||||||
from pathlib import Path
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
|
|
||||||
def logging_init(
|
|
||||||
log_filename="monitor", log_level=logging.INFO, log_dir="./log/", only_file=False
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
初始化日志系统
|
|
||||||
:param log_filename: 日志文件名
|
|
||||||
:param log_level: 日志等级
|
|
||||||
:param log_dir: 日志目录
|
|
||||||
:parma only_file: 是否只保存到日志文件中
|
|
||||||
"""
|
|
||||||
|
|
||||||
# 指定日志文件路径
|
|
||||||
log_dir = Path(log_dir)
|
|
||||||
log_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
log_filepath = log_dir / (log_filename + "_" + str(datetime.now())[:10] + ".txt")
|
|
||||||
|
|
||||||
# 指定日志格式
|
|
||||||
format = "[%(asctime)s] - %(levelname)s: %(message)s"
|
|
||||||
|
|
||||||
# 只保存到日志文件中
|
|
||||||
if only_file:
|
|
||||||
logging.basicConfig(
|
|
||||||
filename=log_filepath,
|
|
||||||
level=log_level,
|
|
||||||
format=format,
|
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
|
||||||
)
|
|
||||||
# 保存到日志文件并输出到终端
|
|
||||||
else:
|
|
||||||
logging.basicConfig(
|
|
||||||
level=log_level,
|
|
||||||
format=format,
|
|
||||||
datefmt="%Y-%m-%d %H:%M:%S",
|
|
||||||
handlers=[
|
|
||||||
logging.FileHandler(log_filepath),
|
|
||||||
logging.StreamHandler(sys.stdout),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
Loading…
Reference in New Issue