baladiwei
/
repostats
1
0
Fork 0
Code Issues Pull Requests Projects Releases 2 Wiki Activity

完成在界面上启动抓取任务

This commit is contained in:
巴拉迪维 2022-04-25 16:40:07 +08:00
parent 3ca7d88759
commit e2d2891ebb
13 changed files with 253 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
assets/admin.js
View File

@ -342,4 +342,40 @@ function searchRepos() {
}
searchForm.submit();
}
function CreateOrUpdateGrafana(repoID) {
$.ajax({
type:"PUT",
url: "/admin/grafana/"+repoID+"/update",
data: {
"type": "gitee"
},
success: function() {
successToast('操作成功')
},
error: function(e) {
errorToast($.parseJSON(e.responseText).message)
}
});
}
function startToGrabRepos() {
$('body').modal('confirm','开始爬取','确认现在就开始爬取数据吗？<br/> 如果需要爬取的仓库较多，过程可能比较久。', function(choice){
if(choice) {
$.ajax({
type:"POST",
url: "/admin/repos/grab",
data: {
"type": "gitee"
},
success: function() {
successToast('操作成功')
},
error: function(e) {
errorToast($.parseJSON(e.responseText).message)
}
});
}//end of if
});
}

96
controller/grafana_c.go
View File

@ -10,9 +10,13 @@ package controller
import (
"encoding/json"
"log"
"net/http"
"repostats/network"
"repostats/schedule"
gitee_storage "repostats/storage/gitee"
"repostats/utils"
"strconv"
"github.com/gin-gonic/gin"
)
@ -64,6 +68,98 @@ func GrafanaToken(ctx *gin.Context) {
return
}
err = network.CreateDatasource(token, utils.DatabaseConifg)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": err.Error(),
})
return
}
datasource, err := network.RetrieveGrafanaDatasource()
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": err.Error(),
})
return
}
err = network.CreateGiteeRepostatsFolder(token)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": err.Error(),
})
return
}
folder, err := network.RetrieveGiteeRepostatsFolder()
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": err.Error(),
})
return
}
err = network.CreateGiteeHomeDashboard(token, folder, datasource)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": err.Error(),
})
return
}
rs, _ := json.Marshal(token)
ctx.JSON(http.StatusOK, rs)
}
//创建或更新制定仓库的 Grafana 视图面板
//
func CreateOrUpdateGrafanaDashboard(ctx *gin.Context) {
strRepoID := ctx.Param("repoID")
repoID, err := strconv.Atoi(strRepoID)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "repoID 参数非法",
})
return
}
if repoID <= 0 {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "repoID 参数非法",
})
return
}
repo, err := gitee_storage.FindRepoByID(repoID)
if err != nil {
log.Println(err)
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "内部错误，请联系管理员！ ",
})
return
}
if repo.IsNilOrEmpty() {
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "找不到指定的仓库，repoID =" + strRepoID,
})
return
}
token, _ := network.RetrieveGrafanaToken()
datasource, _ := network.RetrieveGrafanaDatasource()
folder, _ := network.RetrieveGiteeRepostatsFolder()
err = schedule.CreateOrUpdateGrafanaRepo(repo, token, folder, datasource)
if err != nil {
log.Println(err)
ctx.JSON(http.StatusBadRequest, gin.H{
"message": "内部错误，请联系管理员！ " + err.Error(),
})
return
}
ctx.JSON(http.StatusOK, nil)
}

6
controller/repos_c.go
View File

@ -13,6 +13,7 @@ import (
"net/http"
gitee_model "repostats/model/gitee"
"repostats/network"
"repostats/schedule"
"repostats/storage"
gitee_storage "repostats/storage/gitee"
"repostats/utils"
@ -290,3 +291,8 @@ func RepoDelete(ctx *gin.Context) {
ctx.JSON(http.StatusOK, nil)
}
func StartToGrab(ctx *gin.Context) {
go schedule.StarGiteeJobs(false)
ctx.JSON(http.StatusOK, nil)
}

2
docker/vars.env
View File

@ -8,6 +8,6 @@ PG_SUPER_USER=postgres
PG_SUPER_PWD=DePmoG_123
# Grafana Vars
GR_VERSION=8.3.4
GR_VERSION=8.5.0
GR_CONTAINER_NAME=repostats_gr
GR_LOCAL_PORT=13000

1
network/templates/gitee-home-dashboard.tmpl
View File

@ -10,6 +10,7 @@
"from": "now-30d",
"to": "now"
},
"editable": false,
"panels": [
{
"collapsed": false,

28
network/templates/gitee-repo-dashboard.tmpl
View File

@ -10,6 +10,7 @@
"from": "now-30d",
"to": "now"
},
"editable": false,
"panels": [
{
"collapsed": false,
@ -68,7 +69,6 @@
},
"textMode": "auto"
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -187,7 +187,6 @@
},
"textMode": "auto"
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -306,7 +305,6 @@
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -460,7 +458,6 @@
},
"textMode": "auto"
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -754,7 +751,6 @@
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -1030,7 +1026,6 @@
},
"textMode": "auto"
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -1630,7 +1625,7 @@
],
"metricColumn": "none",
"rawQuery": false,
"rawSql": "SELECT\n $__timeGroupAlias(created_at,$__interval),\n count(id) AS \"Pull Request 数\"\nFROM gitee.pull_requests\nWHERE\n $__timeFilter(created_at)\nGROUP BY 1\nORDER BY 1",
"rawSql": "SELECT\n $__timeGroupAlias(created_at,$__interval),\n count(id) AS \"Pull Request 数\"\nFROM gitee.pull_requests\nWHERE\n $__timeFilter(created_at) and repo_id = {{.repo_id}}\nGROUP BY 1\nORDER BY 1",
"refId": "A",
"select": [
[
@ -1662,7 +1657,17 @@
"name": "$__timeFilter",
"params": [],
"type": "macro"
}
},
{
"datatype": "int8",
"name": "",
"params": [
"repo_id",
"=",
"{{.repo_id}}"
],
"type": "expression"
}
]
}
],
@ -1714,7 +1719,6 @@
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -1870,7 +1874,6 @@
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -2043,7 +2046,6 @@
}
]
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -2161,7 +2163,6 @@
}
]
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -2399,7 +2400,6 @@
}
]
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -2711,7 +2711,6 @@
}
]
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {
@ -2988,7 +2987,6 @@
}
]
},
"pluginVersion": "8.3.4",
"targets": [
{
"datasource": {

4
repostats.go
View File

@ -46,6 +46,8 @@ func main() {
_, err := utils.InitConfig(cmdConfig)
utils.ExitOnError(err)
utils.InitWaitingGruop()
_, err = storage.InitDatabaseService()
utils.ExitOnError(err)
@ -79,6 +81,7 @@ func initRouter(router *gin.Engine) {
admin.PUT("/repos/:repoID/change_state", controller.RepoStateChange)
admin.POST("/repos/:repoID/delete", controller.RepoDelete)
admin.POST("/repos", controller.AddRepo)
admin.POST("/repos/grab", controller.StartToGrab)
admin.GET("/commits", controller.CommitsPage)
admin.POST("/commits/:sha/delete", controller.CommitDelete)
@ -88,6 +91,7 @@ func initRouter(router *gin.Engine) {
admin.GET("/grafana", controller.GrafanaPage)
admin.POST("/grafana/token", controller.GrafanaToken)
admin.PUT("/grafana/:repoID/update", controller.CreateOrUpdateGrafanaDashboard)
admin.GET("/issues", controller.IssuesPage)

86
schedule/gitee_schedule.go
View File

@ -6,24 +6,20 @@ import (
gitee_model "repostats/model/gitee"
"repostats/network"
gitee_storage "repostats/storage/gitee"
"repostats/utils"
"strings"
"time"
"github.com/remeh/sizedwaitgroup"
)
const (
GITEE_SCHEDULER_INTERVAL = 4 * time.Hour
MAX_ROUTINE_NUMBER = 20
)
// 启动 Gitee 定时器
//
func StartGiteeSchedule() error {
ticker := time.NewTicker(GITEE_SCHEDULER_INTERVAL)
ticker := time.NewTicker(utils.GITEE_SCHEDULER_INTERVAL)
for range ticker.C {
log.Println("[RepoStats] Gitee Schedule Start.")
if err := StarGiteeJobs(); err != nil {
if err := StarGiteeJobs(true); err != nil {
log.Printf("[RepoStats] error while doing schedule jobs. %s", err)
}
log.Println("[RepoStats] Gitee Schedule Finish.")
@ -33,7 +29,7 @@ func StartGiteeSchedule() error {
// 启动 Gitee 任务
//
func StarGiteeJobs() error {
func StarGiteeJobs(wait bool) error {
//检查 Grafana Token, Datasource
grafanaToken, err := network.RetrieveGrafanaToken()
@ -64,7 +60,7 @@ func StarGiteeJobs() error {
}
//抓取 Gitee 信息并存储到数据库
wg := sizedwaitgroup.New(MAX_ROUTINE_NUMBER)
wg := utils.WaitingGroup
for _, repo := range repos {
if !repo.EnableCrawl {
continue
@ -72,13 +68,21 @@ func StarGiteeJobs() error {
wg.Add()
go GrabRepo(&wg, repo, giteeToken, grafanaToken, datasource, folder)
}
wg.Wait()
if wait {
wg.Wait()
}
return nil
}
//抓取 Gitee 仓库信息并更新 Grafana 面板
//
func GrabRepo(wg *sizedwaitgroup.SizedWaitGroup, repo gitee_model.Repository,
giteeToken network.OauthToken, grafanaToken network.GrafanaToken, grafanaDatasource network.GrafanaDatasource, grafanaFolder network.GrafanaFolder) error {
defer wg.Done()
if wg != nil {
defer wg.Done()
}
log.Printf("[RepoStats] start to grab [%s]", repo.HTMLURL)
str := strings.Split(repo.FullName, "/")
@ -96,79 +100,85 @@ func GrabRepo(wg *sizedwaitgroup.SizedWaitGroup, repo gitee_model.Repository,
}
var users []gitee_model.User
commits, err := network.GetGiteeCommits(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeCommits %s,%s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(commits); i++ {
commits[i].RepoID = repo.ID
users = append(users, commits[i].Author)
users = append(users, commits[i].Committer)
}
err = gitee_storage.BulkSaveCommits(commits)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveCommits %s, %s", repo.HTMLURL, err)
// return err
if len(commits) > 0 {
err = gitee_storage.BulkSaveCommits(commits)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveCommits %s, %s", repo.HTMLURL, err)
}
}
issues, err := network.GetGiteeIssues(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeIssues %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(issues); i++ {
issues[i].RepoID = int64(repo.ID)
users = append(users, issues[i].User)
}
err = gitee_storage.BulkSaveIssues(issues)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveIssues %s, %s", repo.HTMLURL, err)
// return err
if len(issues) > 0 {
err = gitee_storage.BulkSaveIssues(issues)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveIssues %s, %s", repo.HTMLURL, err)
}
}
prs, err := network.GetGiteePullRequests(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteePullRequests %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(prs); i++ {
prs[i].RepoID = int64(repo.ID)
users = append(users, prs[i].User)
}
usersNeededToSave := gitee_model.RemoveDuplicateUsers(users)
err = gitee_storage.BulkSaveUsers(usersNeededToSave)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveUsers %s, %s", repo.HTMLURL, err)
// return err
if len(usersNeededToSave) > 0 {
err = gitee_storage.BulkSaveUsers(usersNeededToSave)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveUsers %s, %s", repo.HTMLURL, err)
}
}
err = gitee_storage.BulkSavePullRequests(prs)
if err != nil {
log.Printf("[RepoStats] failed during BulkSavePullRequests %s,%s", repo.HTMLURL, err)
// return err
if len(prs) > 0 {
err = gitee_storage.BulkSavePullRequests(prs)
if err != nil {
log.Printf("[RepoStats] failed during BulkSavePullRequests %s,%s", repo.HTMLURL, err)
}
}
stargazers, err := network.GetGiteeStargazers(str[0], str[1])
if err != nil {
log.Printf("[RepoStats] failed during GetGiteeStargazers %s, %s", repo.HTMLURL, err)
// return err
}
for i := 0; i < len(stargazers); i++ {
stargazers[i].RepoID = int64(repo.ID)
}
err = gitee_storage.BulkSaveStargazers(stargazers)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveStargazers %s, %s", repo.HTMLURL, err)
// return err
if len(stargazers) > 0 {
err = gitee_storage.BulkSaveStargazers(stargazers)
if err != nil {
log.Printf("[RepoStats] failed during BulkSaveStargazers %s, %s", repo.HTMLURL, err)
}
}
err = network.CreateGiteeRepoDashboard(grafanaToken, grafanaFolder, grafanaDatasource, repo)
err = CreateOrUpdateGrafanaRepo(repo, grafanaToken, grafanaFolder, grafanaDatasource)
if err != nil {
log.Printf("[RepoStats] failed during CreateGiteeRepoDashboard %s, %s", repo.HTMLURL, err)
// return err
}
log.Printf("[RepoStats] finish to grab [%s]", repo.HTMLURL)
return nil
}
//创建或更新 Grafana 的项目视图面板
//
func CreateOrUpdateGrafanaRepo(repo gitee_model.Repository, token network.GrafanaToken,
folder network.GrafanaFolder, datasource network.GrafanaDatasource) error {
return network.CreateGiteeRepoDashboard(token, folder, datasource, repo)
}

2
schedule/gitee_schedule_test.go
View File

@ -20,7 +20,7 @@ func TestStarGiteeJobs(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := StarGiteeJobs(); (err != nil) != tt.wantErr {
if err := StarGiteeJobs(true); (err != nil) != tt.wantErr {
t.Errorf("StarGiteeJobs() error = %v, wantErr %v", err, tt.wantErr)
}
})

24
storage/gitee/commit_s.go
View File

@ -13,6 +13,8 @@ import (
"repostats/storage"
)
const max_array_size = 60000
var commitQueryPrefix = `SELECT c.author_name AS "author.name",c.author_email AS "author.email",
c.author_name AS "commit.author.name", c.author_email AS "commit.author.email", c.author_date AS "commit.author.date",
c.committer_name AS "committer.name",c.committer_email AS "committer.email",
@ -27,6 +29,13 @@ func BulkSaveCommits(commits []gitee_model.Commit) error {
ON CONFLICT (repo_id,sha) DO UPDATE SET repo_id=EXCLUDED.repo_id,html_url=EXCLUDED.html_url,
author_name=EXCLUDED.author_name,author_email=EXCLUDED.author_email,author_date=EXCLUDED.author_date,committer_name=EXCLUDED.committer_name,
committer_email=EXCLUDED.committer_email,committer_date=EXCLUDED.committer_date,detail_message=EXCLUDED.detail_message,tree=EXCLUDED.tree`
if len(commits) > max_array_size {
nc := splitCommitsArray(commits)
for i := 0; i < len(nc); i++ {
storage.DbNamedExec(query, nc[i])
}
return nil
}
return storage.DbNamedExec(query, commits)
}
@ -103,3 +112,18 @@ func DeleteCommitBySha(sha string) error {
query := `DELETE FROM gitee.commits WHERE sha = $1`
return storage.DbExec(query, sha)
}
func splitCommitsArray(slice []gitee_model.Commit) [][]gitee_model.Commit {
var chunks [][]gitee_model.Commit
for i := 0; i < len(slice); i += max_array_size {
end := i + max_array_size
if end > len(slice) {
end = len(slice)
}
chunks = append(chunks, slice[i:end])
}
return chunks
}

2
templates/issues.html
View File

@ -29,7 +29,7 @@
</thead>
<tbody>
{{range $i,$iss := .issues}}
{{if eq "closed" $iss.State}}<tr class="negative">{{else if eq "progressing" $iss.State}}<tr class="positive">{{else}}<tr>{{end}}
{{if eq "closed" $iss.State}}<tr class="positive">{{else if eq "rejected" $iss.State}}<tr class="positive">{{else}}<tr>{{end}}
<td class="center aligned collapsing">{{add $i 1}}</td>
<td class="center aligned collapsing">Gitee</td>
<td class="center aligned collapsing">{{$iss.Number}}</td>

9
templates/repos.html
View File

@ -28,7 +28,10 @@
</th>
<th colspan="8">
<p>新增、删除或禁止爬取仓库数据，不会删除 Gitee 数据，但会影响 RepoStats 统计结果</p>
<p><button onclick="showAddRepoModal()" class="ui mini positive button">添加仓库</button></p>
<p>
<button onclick="showAddRepoModal()" class="ui mini positive button">添加仓库</button>
<button onclick="startToGrabRepos()" class="ui mini red button">开始爬取</button>
</p>
</th>
</tr>
<tr>
@ -69,7 +72,9 @@
<a href="javascript:disableCrawl('{{$r.ID}}','{{$r.HumanName}}')">禁止爬取</a>
{{else}}
<a href="javascript:enableCrawl('{{$r.ID}}','{{$r.HumanName}}')">开启爬取</a>
{{end}} | <a href="javascript:deleteRepo('{{$r.ID}}','{{$r.HumanName}}')">删除</a></td>
{{end}}
| <a href="javascript:CreateOrUpdateGrafana('{{$r.ID}}')">更新 Grafana 面板</a>
| <a href="javascript:deleteRepo('{{$r.ID}}','{{$r.HumanName}}')">删除</a></td>
</tr>
{{end}}
</tbody>

17
utils/utils.go
View File

@ -16,13 +16,26 @@ import (
"path/filepath"
"reflect"
"strings"
"time"
"github.com/remeh/sizedwaitgroup"
)
const (
MAX_ROUTINE_NUMBER = 25
GITEE_SCHEDULER_INTERVAL = 4 * time.Hour
)
var (
Version = "1.0"
Build = "2204111911"
Version = "1.0"
Build = "2204111911"
WaitingGroup = sizedwaitgroup.SizedWaitGroup{}
)
func InitWaitingGruop() {
WaitingGroup = sizedwaitgroup.New(MAX_ROUTINE_NUMBER)
}
func ExitOnError(err error) {
if err != nil {
log.Fatalln(err)