新增用户时增加桶
This commit is contained in:
parent
b21e15763d
commit
fa797c1774
|
|
@ -33,5 +33,17 @@
|
||||||
},
|
},
|
||||||
"cloudreamStorage": {
|
"cloudreamStorage": {
|
||||||
"url": "http://localhost:32010"
|
"url": "http://localhost:32010"
|
||||||
|
},
|
||||||
|
"pcmHub": [
|
||||||
|
{
|
||||||
|
"name": "OpenI",
|
||||||
|
"APIKey": "",
|
||||||
|
"URL": "https://qaygilqyldcc.test.jointcloud.net:443"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ModelArts",
|
||||||
|
"APIKey": "",
|
||||||
|
"URL": "https://fwoofrxezczr.test.jointcloud.net:443"
|
||||||
}
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -1,36 +1,15 @@
|
||||||
{
|
{
|
||||||
"inputs": [
|
"userID": 1,
|
||||||
{
|
"packageID": 1126,
|
||||||
"url": "https://pcm2-bucket2.obs.cn-north-4.myhuaweicloud.com:443/cds/blocks/50/Full50B464DB2FDDC29D0380D9FFAB6D944FAF5C7624955D757939280590F01F3ECD?AWSAccessKeyId=CANMDYKXIWRDR0IYDB32&Expires=1737685829&Signature=m4IdspV4RQcJgm4Ls9%2Fosaxr2o4%3D"
|
"uploadParams": {
|
||||||
},
|
"dataType": "dataset",
|
||||||
{
|
"uploadInfo": {
|
||||||
"url": "https://pcm2-bucket2.obs.cn-north-4.myhuaweicloud.com:443/cds/blocks/C0/FullC036CBB7553A909F8B8877D4461924307F27ECB66CFF928EEEAFD569C3887E29?AWSAccessKeyId=CANMDYKXIWRDR0IYDB32&Expires=1737685829&Signature=pQTrQSPUw3k4h4Q6yfZp3a8wQ6M%3D"
|
"type": "url",
|
||||||
},
|
"url": "https://openi.pcl.ac.cn/somunslotus/meta-weight-net/grampus/train-job/501239",
|
||||||
{
|
"targetClusters": [
|
||||||
"url": "https://pcm2-bucket2.obs.cn-north-4.myhuaweicloud.com:443/cds/blocks/54/Full543F38D9F524238AC0239263AA0DD4B4328763818EA98A7A5F72E59748FDA27A?AWSAccessKeyId=CANMDYKXIWRDR0IYDB32&Expires=1737685829&Signature=voCaGtEJ6z%2Bp8TWl8hJw4ytatnE%3D"
|
"1865927992266461184"
|
||||||
}
|
|
||||||
],
|
|
||||||
"outputs": [
|
|
||||||
"/tmp/IGsnvEIyru_0",
|
|
||||||
"/tmp/IGsnvEIyru_1",
|
|
||||||
"/tmp/IGsnvEIyru_2"
|
|
||||||
],
|
|
||||||
"coefs": [
|
|
||||||
[
|
|
||||||
186,
|
|
||||||
2,
|
|
||||||
185
|
|
||||||
],
|
|
||||||
[
|
|
||||||
186,
|
|
||||||
3,
|
|
||||||
184
|
|
||||||
],
|
|
||||||
[
|
|
||||||
1,
|
|
||||||
1,
|
|
||||||
1
|
|
||||||
]
|
]
|
||||||
],
|
},
|
||||||
"chunkSize": 5242880
|
"dataName": "501239"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -149,7 +149,7 @@ func (db *AccessDB) GetUserBySsoID(ctx SQLContext, id string) (cdssdk.UserID, er
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *AccessDB) AddUser(ctx SQLContext, user sch.User) error {
|
func (db *AccessDB) AddUser(ctx SQLContext, user sch.User, buckets []sch.Bucket) error {
|
||||||
// 根据ssoID查询数据是否存在
|
// 根据ssoID查询数据是否存在
|
||||||
var existUser sch.User
|
var existUser sch.User
|
||||||
err := ctx.Table("users").Where("sso_id = ?", user.SsoID).Find(&existUser).Error
|
err := ctx.Table("users").Where("sso_id = ?", user.SsoID).Find(&existUser).Error
|
||||||
|
|
@ -160,9 +160,33 @@ func (db *AccessDB) AddUser(ctx SQLContext, user sch.User) error {
|
||||||
return fmt.Errorf("user already exists")
|
return fmt.Errorf("user already exists")
|
||||||
}
|
}
|
||||||
|
|
||||||
return ctx.Table("users").Create(&user).Error
|
tx := ctx.Begin()
|
||||||
|
err = tx.Table("users").Create(&user).Error
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
err = tx.Table("bucket").Create(&bucket).Error
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tx.Commit()
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *AccessDB) UpdateUser(ctx SQLContext, id string, username string) error {
|
func (db *AccessDB) UpdateUser(ctx SQLContext, id string, username string) error {
|
||||||
return ctx.Table("users").Where("sso_id = ?", id).Update("username", username).Error
|
return ctx.Table("users").Where("sso_id = ?", id).Update("username", username).Error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (db *AccessDB) GetBucketByUserID(ctx SQLContext, userID cdssdk.UserID, dataType string) (*sch.Bucket, error) {
|
||||||
|
var ret sch.Bucket
|
||||||
|
if err := ctx.Table("bucket").Where("user_id = ? and data_type = ?", userID, dataType).First(&ret).Error; err != nil {
|
||||||
|
return &ret, err
|
||||||
|
}
|
||||||
|
return &ret, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -2125,3 +2125,74 @@ Key: 'QueryUploadedReq.UserID' Error:Field validation for 'UserID' failed on the
|
||||||
2025-01-23 16:42:28 [DEBU] submitting job
|
2025-01-23 16:42:28 [DEBU] submitting job
|
||||||
2025-01-25 08:01:27 [INFO] start serving http at: :7891
|
2025-01-25 08:01:27 [INFO] start serving http at: :7891
|
||||||
2025-01-25 08:02:56 [INFO] start serving http at: :7891
|
2025-01-25 08:02:56 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 10:44:47 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 10:49:31 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 10:51:33 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 10:53:18 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 10:57:22 [WARN] [HTTP:JobSet.QueryUploaded] getting service list: failed to query uploaded data: Get "http://localhost:32010/object/listByPath?isPrefix=true&packageID=1093&path=&userID=1": dial tcp [::1]:32010: connectex: No connection could be made because the target machine actively refused it.
|
||||||
|
2025-02-10 11:13:36 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 11:28:40 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 11:30:25 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 11:35:21 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 14:51:59 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 14:54:33 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 14:56:11 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 14:57:00 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 15:54:36 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 15:57:16 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 15:58:50 [WARN] get access requests: Error 1364 (HY000): Field 'password' doesn't have a default value
|
||||||
|
2025-02-10 16:00:04 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 16:02:07 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 16:03:47 [INFO] start serving http at: :7891
|
||||||
|
2025-02-10 16:03:56 [WARN] get access requests: user already exists
|
||||||
|
2025-02-11 09:59:01 [INFO] start serving http at: :7891
|
||||||
|
2025-02-11 09:59:48 [DEBU] submitting job
|
||||||
|
2025-02-11 10:02:09 [INFO] start serving http at: :7891
|
||||||
|
2025-02-11 10:02:17 [DEBU] submitting job
|
||||||
|
2025-02-11 10:04:34 [INFO] start serving http at: :7891
|
||||||
|
2025-02-11 10:04:36 [DEBU] submitting job
|
||||||
|
2025-02-11 10:35:11 [INFO] start serving http at: :7891
|
||||||
|
2025-02-11 10:35:20 [DEBU] submitting job
|
||||||
|
2025-02-11 10:42:38 [INFO] start serving http at: :7891
|
||||||
|
2025-02-11 10:42:49 [DEBU] submitting job
|
||||||
|
2025-02-11 10:43:31 [ERRO] create task: Post "https://comnet.jointcloud.net/pcm/v1/schedule/createTask": dial tcp: lookup comnet.jointcloud.net: no such host
|
||||||
|
2025-02-11 10:43:31 [INFO] jobID: %s change state from %s to %s0&{0xc0000ca8f0 1} &{0xc000324d00}
|
||||||
|
2025-02-11 10:43:31 [INFO] [JobID:0] state changed: *state2.PCMJobCreate -> *state.Completed
|
||||||
|
2025-02-11 10:43:31 [INFO] [JobID:0] [LastState:*state2.PCMJobCreate] job failed with: create task: Post "https://comnet.jointcloud.net/pcm/v1/schedule/createTask": dial tcp: lookup comnet.jointcloud.net: no such host
|
||||||
|
2025-02-11 10:43:31 [INFO] job set 0 completed
|
||||||
|
2025-02-11 10:43:57 [DEBU] submitting job
|
||||||
|
2025-02-11 10:44:26 [INFO] jobID: %s change state from %s to %s1&{0xc0000cb340 1} &{1 1889139808467423233 [{model 1070 EFILE [1777240145309732864]}] }
|
||||||
|
2025-02-11 10:44:26 [INFO] [JobID:1] state changed: *state2.PCMJobCreate -> *state2.DataSchedule
|
||||||
|
2025-02-11 10:44:31 [ERRO] schedule data: code: 500, message:
|
||||||
|
2025-02-11 10:44:31 [INFO] jobID: %s change state from %s to %s1&{1 1889139808467423233 [{model 1070 EFILE [1777240145309732864]}] } &{1889139808467423233 schedule data: code: 500, message: }
|
||||||
|
2025-02-11 10:44:31 [INFO] [JobID:1] state changed: *state2.DataSchedule -> *state2.PCMJobCancel
|
||||||
|
2025-02-11 10:44:31 [ERRO] unknow response content type:
|
||||||
|
2025-02-11 10:44:31 [INFO] jobID: %s change state from %s to %s1&{1889139808467423233 schedule data: code: 500, message: } &{0xc00023a010}
|
||||||
|
2025-02-11 10:44:31 [INFO] [JobID:1] state changed: *state2.PCMJobCancel -> *state.Completed
|
||||||
|
2025-02-11 10:44:31 [INFO] [JobID:1] [LastState:*state2.PCMJobCancel] job failed with: unknow response content type:
|
||||||
|
2025-02-11 10:44:31 [INFO] job set 1 completed
|
||||||
|
2025-02-12 15:15:26 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 15:21:02 [WARN] get access requests: failed to create user: unknow response content type: text/plain, status: 404, body(prefix): 404 page not found
|
||||||
|
2025-02-12 15:21:30 [WARN] get access requests: failed to create user: unknow response content type: text/plain, status: 404, body(prefix): 404 page not found
|
||||||
|
2025-02-12 15:21:57 [WARN] get access requests: failed to create user: unknow response content type: text/plain, status: 404, body(prefix): 404 page not found
|
||||||
|
2025-02-12 15:23:12 [WARN] get access requests: failed to create user: unknow response content type: text/plain, status: 404, body(prefix): 404 page not found
|
||||||
|
2025-02-12 15:57:38 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:29:40 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:42:18 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:44:50 [DEBU] uploading job
|
||||||
|
2025-02-12 16:49:42 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:52:19 [DEBU] uploading job
|
||||||
|
2025-02-12 16:52:23 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:52:45 [DEBU] uploading job
|
||||||
|
2025-02-12 16:54:10 [ERRO] upload data: code: 500, message:
|
||||||
|
2025-02-12 16:54:10 [INFO] jobID: %s change state from %s to %s0&{1 0xc0005b9ae0 code {1 0}} &{0xc0001fa0a0}
|
||||||
|
2025-02-12 16:54:10 [INFO] [JobID:0] state changed: *state2.DataUpload -> *state.Completed
|
||||||
|
2025-02-12 16:54:10 [INFO] [JobID:0] [LastState:*state2.DataUpload] job failed with: upload data: code: 500, message:
|
||||||
|
2025-02-12 16:54:10 [INFO] job set 0 completed
|
||||||
|
2025-02-12 16:54:18 [INFO] start serving http at: :7891
|
||||||
|
2025-02-12 16:54:51 [DEBU] uploading job
|
||||||
|
2025-02-12 16:56:48 [ERRO] upload data: code: 500, message:
|
||||||
|
2025-02-12 16:56:48 [INFO] jobID: %s change state from %s to %s0&{1 0xc0002480a0 code {1 0}} &{0xc00067c700}
|
||||||
|
2025-02-12 16:56:48 [INFO] [JobID:0] state changed: *state2.DataUpload -> *state.Completed
|
||||||
|
2025-02-12 16:56:48 [INFO] [JobID:0] [LastState:*state2.DataUpload] job failed with: upload data: code: 500, message:
|
||||||
|
2025-02-12 16:56:48 [INFO] job set 0 completed
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,727 @@
|
||||||
|
// +build ignore
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
_flag "flag"
|
||||||
|
_fmt "fmt"
|
||||||
|
_ioutil "io/ioutil"
|
||||||
|
_log "log"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
_filepath "path/filepath"
|
||||||
|
_sort "sort"
|
||||||
|
"strconv"
|
||||||
|
_strings "strings"
|
||||||
|
"syscall"
|
||||||
|
_tabwriter "text/tabwriter"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// Use local types and functions in order to avoid name conflicts with additional magefiles.
|
||||||
|
type arguments struct {
|
||||||
|
Verbose bool // print out log statements
|
||||||
|
List bool // print out a list of targets
|
||||||
|
Help bool // print out help for a specific target
|
||||||
|
Timeout time.Duration // set a timeout to running the targets
|
||||||
|
Args []string // args contain the non-flag command-line arguments
|
||||||
|
}
|
||||||
|
|
||||||
|
parseBool := func(env string) bool {
|
||||||
|
val := os.Getenv(env)
|
||||||
|
if val == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b, err := strconv.ParseBool(val)
|
||||||
|
if err != nil {
|
||||||
|
_log.Printf("warning: environment variable %s is not a valid bool value: %v", env, val)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
parseDuration := func(env string) time.Duration {
|
||||||
|
val := os.Getenv(env)
|
||||||
|
if val == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(val)
|
||||||
|
if err != nil {
|
||||||
|
_log.Printf("warning: environment variable %s is not a valid duration value: %v", env, val)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
args := arguments{}
|
||||||
|
fs := _flag.FlagSet{}
|
||||||
|
fs.SetOutput(os.Stdout)
|
||||||
|
|
||||||
|
// default flag set with ExitOnError and auto generated PrintDefaults should be sufficient
|
||||||
|
fs.BoolVar(&args.Verbose, "v", parseBool("MAGEFILE_VERBOSE"), "show verbose output when running targets")
|
||||||
|
fs.BoolVar(&args.List, "l", parseBool("MAGEFILE_LIST"), "list targets for this binary")
|
||||||
|
fs.BoolVar(&args.Help, "h", parseBool("MAGEFILE_HELP"), "print out help for a specific target")
|
||||||
|
fs.DurationVar(&args.Timeout, "t", parseDuration("MAGEFILE_TIMEOUT"), "timeout in duration parsable format (e.g. 5m30s)")
|
||||||
|
fs.Usage = func() {
|
||||||
|
_fmt.Fprintf(os.Stdout, `
|
||||||
|
%s [options] [target]
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
-l list targets in this binary
|
||||||
|
-h show this help
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-h show description of a target
|
||||||
|
-t <string>
|
||||||
|
timeout in duration parsable format (e.g. 5m30s)
|
||||||
|
-v show verbose output when running targets
|
||||||
|
`[1:], _filepath.Base(os.Args[0]))
|
||||||
|
}
|
||||||
|
if err := fs.Parse(os.Args[1:]); err != nil {
|
||||||
|
// flag will have printed out an error already.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
args.Args = fs.Args()
|
||||||
|
if args.Help && len(args.Args) == 0 {
|
||||||
|
fs.Usage()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// color is ANSI color type
|
||||||
|
type color int
|
||||||
|
|
||||||
|
// If you add/change/remove any items in this constant,
|
||||||
|
// you will need to run "stringer -type=color" in this directory again.
|
||||||
|
// NOTE: Please keep the list in an alphabetical order.
|
||||||
|
const (
|
||||||
|
black color = iota
|
||||||
|
red
|
||||||
|
green
|
||||||
|
yellow
|
||||||
|
blue
|
||||||
|
magenta
|
||||||
|
cyan
|
||||||
|
white
|
||||||
|
brightblack
|
||||||
|
brightred
|
||||||
|
brightgreen
|
||||||
|
brightyellow
|
||||||
|
brightblue
|
||||||
|
brightmagenta
|
||||||
|
brightcyan
|
||||||
|
brightwhite
|
||||||
|
)
|
||||||
|
|
||||||
|
// AnsiColor are ANSI color codes for supported terminal colors.
|
||||||
|
var ansiColor = map[color]string{
|
||||||
|
black: "\u001b[30m",
|
||||||
|
red: "\u001b[31m",
|
||||||
|
green: "\u001b[32m",
|
||||||
|
yellow: "\u001b[33m",
|
||||||
|
blue: "\u001b[34m",
|
||||||
|
magenta: "\u001b[35m",
|
||||||
|
cyan: "\u001b[36m",
|
||||||
|
white: "\u001b[37m",
|
||||||
|
brightblack: "\u001b[30;1m",
|
||||||
|
brightred: "\u001b[31;1m",
|
||||||
|
brightgreen: "\u001b[32;1m",
|
||||||
|
brightyellow: "\u001b[33;1m",
|
||||||
|
brightblue: "\u001b[34;1m",
|
||||||
|
brightmagenta: "\u001b[35;1m",
|
||||||
|
brightcyan: "\u001b[36;1m",
|
||||||
|
brightwhite: "\u001b[37;1m",
|
||||||
|
}
|
||||||
|
|
||||||
|
const _color_name = "blackredgreenyellowbluemagentacyanwhitebrightblackbrightredbrightgreenbrightyellowbrightbluebrightmagentabrightcyanbrightwhite"
|
||||||
|
|
||||||
|
var _color_index = [...]uint8{0, 5, 8, 13, 19, 23, 30, 34, 39, 50, 59, 70, 82, 92, 105, 115, 126}
|
||||||
|
|
||||||
|
colorToLowerString := func (i color) string {
|
||||||
|
if i < 0 || i >= color(len(_color_index)-1) {
|
||||||
|
return "color(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||||
|
}
|
||||||
|
return _color_name[_color_index[i]:_color_index[i+1]]
|
||||||
|
}
|
||||||
|
|
||||||
|
// ansiColorReset is an ANSI color code to reset the terminal color.
|
||||||
|
const ansiColorReset = "\033[0m"
|
||||||
|
|
||||||
|
// defaultTargetAnsiColor is a default ANSI color for colorizing targets.
|
||||||
|
// It is set to Cyan as an arbitrary color, because it has a neutral meaning
|
||||||
|
var defaultTargetAnsiColor = ansiColor[cyan]
|
||||||
|
|
||||||
|
getAnsiColor := func(color string) (string, bool) {
|
||||||
|
colorLower := _strings.ToLower(color)
|
||||||
|
for k, v := range ansiColor {
|
||||||
|
colorConstLower := colorToLowerString(k)
|
||||||
|
if colorConstLower == colorLower {
|
||||||
|
return v, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Terminals which don't support color:
|
||||||
|
// TERM=vt100
|
||||||
|
// TERM=cygwin
|
||||||
|
// TERM=xterm-mono
|
||||||
|
var noColorTerms = map[string]bool{
|
||||||
|
"vt100": false,
|
||||||
|
"cygwin": false,
|
||||||
|
"xterm-mono": false,
|
||||||
|
}
|
||||||
|
|
||||||
|
// terminalSupportsColor checks if the current console supports color output
|
||||||
|
//
|
||||||
|
// Supported:
|
||||||
|
// linux, mac, or windows's ConEmu, Cmder, putty, git-bash.exe, pwsh.exe
|
||||||
|
// Not supported:
|
||||||
|
// windows cmd.exe, powerShell.exe
|
||||||
|
terminalSupportsColor := func() bool {
|
||||||
|
envTerm := os.Getenv("TERM")
|
||||||
|
if _, ok := noColorTerms[envTerm]; ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// enableColor reports whether the user has requested to enable a color output.
|
||||||
|
enableColor := func() bool {
|
||||||
|
b, _ := strconv.ParseBool(os.Getenv("MAGEFILE_ENABLE_COLOR"))
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// targetColor returns the ANSI color which should be used to colorize targets.
|
||||||
|
targetColor := func() string {
|
||||||
|
s, exists := os.LookupEnv("MAGEFILE_TARGET_COLOR")
|
||||||
|
if exists == true {
|
||||||
|
if c, ok := getAnsiColor(s); ok == true {
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultTargetAnsiColor
|
||||||
|
}
|
||||||
|
|
||||||
|
// store the color terminal variables, so that the detection isn't repeated for each target
|
||||||
|
var enableColorValue = enableColor() && terminalSupportsColor()
|
||||||
|
var targetColorValue = targetColor()
|
||||||
|
|
||||||
|
printName := func(str string) string {
|
||||||
|
if enableColorValue {
|
||||||
|
return _fmt.Sprintf("%s%s%s", targetColorValue, str, ansiColorReset)
|
||||||
|
} else {
|
||||||
|
return str
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
list := func() error {
|
||||||
|
|
||||||
|
targets := map[string]string{
|
||||||
|
"amD64": "[配置项]设置编译架构为amd64",
|
||||||
|
"advisor": "",
|
||||||
|
"all": "",
|
||||||
|
"bin": "",
|
||||||
|
"client": "",
|
||||||
|
"collector": "",
|
||||||
|
"confs": "",
|
||||||
|
"executor": "",
|
||||||
|
"linux": "[配置项]设置编译平台为linux",
|
||||||
|
"manager": "",
|
||||||
|
"scheduleMiddleware": "",
|
||||||
|
"scripts": "",
|
||||||
|
"win": "[配置项]设置编译平台为windows",
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(targets))
|
||||||
|
for name := range targets {
|
||||||
|
keys = append(keys, name)
|
||||||
|
}
|
||||||
|
_sort.Strings(keys)
|
||||||
|
|
||||||
|
_fmt.Println("Targets:")
|
||||||
|
w := _tabwriter.NewWriter(os.Stdout, 0, 4, 4, ' ', 0)
|
||||||
|
for _, name := range keys {
|
||||||
|
_fmt.Fprintf(w, " %v\t%v\n", printName(name), targets[name])
|
||||||
|
}
|
||||||
|
err := w.Flush()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var ctx context.Context
|
||||||
|
ctxCancel := func(){}
|
||||||
|
|
||||||
|
// by deferring in a closure, we let the cancel function get replaced
|
||||||
|
// by the getContext function.
|
||||||
|
defer func() {
|
||||||
|
ctxCancel()
|
||||||
|
}()
|
||||||
|
|
||||||
|
getContext := func() (context.Context, func()) {
|
||||||
|
if ctx == nil {
|
||||||
|
if args.Timeout != 0 {
|
||||||
|
ctx, ctxCancel = context.WithTimeout(context.Background(), args.Timeout)
|
||||||
|
} else {
|
||||||
|
ctx, ctxCancel = context.WithCancel(context.Background())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx, ctxCancel
|
||||||
|
}
|
||||||
|
|
||||||
|
runTarget := func(logger *_log.Logger, fn func(context.Context) error) interface{} {
|
||||||
|
var err interface{}
|
||||||
|
ctx, cancel := getContext()
|
||||||
|
d := make(chan interface{})
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
err := recover()
|
||||||
|
d <- err
|
||||||
|
}()
|
||||||
|
err := fn(ctx)
|
||||||
|
d <- err
|
||||||
|
}()
|
||||||
|
sigCh := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigCh, syscall.SIGINT)
|
||||||
|
select {
|
||||||
|
case <-sigCh:
|
||||||
|
logger.Println("cancelling mage targets, waiting up to 5 seconds for cleanup...")
|
||||||
|
cancel()
|
||||||
|
cleanupCh := time.After(5 * time.Second)
|
||||||
|
|
||||||
|
select {
|
||||||
|
// target exited by itself
|
||||||
|
case err = <-d:
|
||||||
|
return err
|
||||||
|
// cleanup timeout exceeded
|
||||||
|
case <-cleanupCh:
|
||||||
|
return _fmt.Errorf("cleanup timeout exceeded")
|
||||||
|
// second SIGINT received
|
||||||
|
case <-sigCh:
|
||||||
|
logger.Println("exiting mage")
|
||||||
|
return _fmt.Errorf("exit forced")
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
cancel()
|
||||||
|
e := ctx.Err()
|
||||||
|
_fmt.Printf("ctx err: %v\n", e)
|
||||||
|
return e
|
||||||
|
case err = <-d:
|
||||||
|
// we intentionally don't cancel the context here, because
|
||||||
|
// the next target will need to run with the same context.
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// This is necessary in case there aren't any targets, to avoid an unused
|
||||||
|
// variable error.
|
||||||
|
_ = runTarget
|
||||||
|
|
||||||
|
handleError := func(logger *_log.Logger, err interface{}) {
|
||||||
|
if err != nil {
|
||||||
|
logger.Printf("Error: %+v\n", err)
|
||||||
|
type code interface {
|
||||||
|
ExitStatus() int
|
||||||
|
}
|
||||||
|
if c, ok := err.(code); ok {
|
||||||
|
os.Exit(c.ExitStatus())
|
||||||
|
}
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = handleError
|
||||||
|
|
||||||
|
// Set MAGEFILE_VERBOSE so mg.Verbose() reflects the flag value.
|
||||||
|
if args.Verbose {
|
||||||
|
os.Setenv("MAGEFILE_VERBOSE", "1")
|
||||||
|
} else {
|
||||||
|
os.Setenv("MAGEFILE_VERBOSE", "0")
|
||||||
|
}
|
||||||
|
|
||||||
|
_log.SetFlags(0)
|
||||||
|
if !args.Verbose {
|
||||||
|
_log.SetOutput(_ioutil.Discard)
|
||||||
|
}
|
||||||
|
logger := _log.New(os.Stderr, "", 0)
|
||||||
|
if args.List {
|
||||||
|
if err := list(); err != nil {
|
||||||
|
_log.Println(err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if args.Help {
|
||||||
|
if len(args.Args) < 1 {
|
||||||
|
logger.Println("no target specified")
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
switch _strings.ToLower(args.Args[0]) {
|
||||||
|
case "amd64":
|
||||||
|
_fmt.Println("[配置项]设置编译架构为amd64")
|
||||||
|
_fmt.Println()
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage amd64\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "advisor":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage advisor\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "all":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage all\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "bin":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage bin\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "client":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage client\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "collector":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage collector\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "confs":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage confs\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "executor":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage executor\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "linux":
|
||||||
|
_fmt.Println("[配置项]设置编译平台为linux")
|
||||||
|
_fmt.Println()
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage linux\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "manager":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage manager\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "schedulemiddleware":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage schedulemiddleware\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "scripts":
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage scripts\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
case "win":
|
||||||
|
_fmt.Println("[配置项]设置编译平台为windows")
|
||||||
|
_fmt.Println()
|
||||||
|
|
||||||
|
_fmt.Print("Usage:\n\n\tmage win\n\n")
|
||||||
|
var aliases []string
|
||||||
|
if len(aliases) > 0 {
|
||||||
|
_fmt.Printf("Aliases: %s\n\n", _strings.Join(aliases, ", "))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
logger.Printf("Unknown target: %q\n", args.Args[0])
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(args.Args) < 1 {
|
||||||
|
if err := list(); err != nil {
|
||||||
|
logger.Println("Error:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for x := 0; x < len(args.Args); {
|
||||||
|
target := args.Args[x]
|
||||||
|
x++
|
||||||
|
|
||||||
|
// resolve aliases
|
||||||
|
switch _strings.ToLower(target) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
switch _strings.ToLower(target) {
|
||||||
|
|
||||||
|
case "amd64":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"AMD64\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "AMD64")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
AMD64()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "advisor":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Advisor\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Advisor")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Advisor()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "all":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"All\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "All")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return All()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "bin":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Bin\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Bin")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Bin()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "client":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Client\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Client")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Client()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "collector":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Collector\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Collector")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Collector()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "confs":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Confs\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Confs")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Confs()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "executor":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Executor\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Executor")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Executor()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "linux":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Linux\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Linux")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
Linux()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "manager":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Manager\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Manager")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Manager()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "schedulemiddleware":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"ScheduleMiddleware\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "ScheduleMiddleware")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return ScheduleMiddleware()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "scripts":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Scripts\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Scripts")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
return Scripts()
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
case "win":
|
||||||
|
expected := x + 0
|
||||||
|
if expected > len(args.Args) {
|
||||||
|
// note that expected and args at this point include the arg for the target itself
|
||||||
|
// so we subtract 1 here to show the number of args without the target.
|
||||||
|
logger.Printf("not enough arguments for target \"Win\", expected %v, got %v\n", expected-1, len(args.Args)-1)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
if args.Verbose {
|
||||||
|
logger.Println("Running target:", "Win")
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapFn := func(ctx context.Context) error {
|
||||||
|
Win()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ret := runTarget(logger, wrapFn)
|
||||||
|
handleError(logger, ret)
|
||||||
|
|
||||||
|
default:
|
||||||
|
logger.Printf("Unknown target specified: %q\n", target)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
pcmhub "gitlink.org.cn/JointCloud/pcm-hub/config"
|
||||||
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
||||||
"gitlink.org.cn/cloudream/common/sdks/blockchain"
|
"gitlink.org.cn/cloudream/common/sdks/blockchain"
|
||||||
sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler"
|
sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler"
|
||||||
|
|
@ -17,6 +18,7 @@ type Config struct {
|
||||||
Uploader uploadersdk.Config `json:"uploader"`
|
Uploader uploadersdk.Config `json:"uploader"`
|
||||||
BlockChain blockchain.Config `json:"blockChain"`
|
BlockChain blockchain.Config `json:"blockChain"`
|
||||||
CloudreamStorage cdsapi.Config `json:"cloudreamStorage"`
|
CloudreamStorage cdsapi.Config `json:"cloudreamStorage"`
|
||||||
|
PCMHub []pcmhub.PlatformConfig `json:"pcmHub"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg Config
|
var cfg Config
|
||||||
|
|
|
||||||
|
|
@ -72,9 +72,11 @@ func (s *DataUpload) do(rtx jobmgr.JobStateRunContext) error {
|
||||||
req := uploadersdk.UploadReq{
|
req := uploadersdk.UploadReq{
|
||||||
DataType: s.dataType,
|
DataType: s.dataType,
|
||||||
Source: &uploadersdk.UrlSource{
|
Source: &uploadersdk.UrlSource{
|
||||||
|
Type: sch.StorageTypeURL,
|
||||||
Url: info.Url,
|
Url: info.Url,
|
||||||
},
|
},
|
||||||
Target: &uploadersdk.UrlTarget{
|
Target: &uploadersdk.UrlTarget{
|
||||||
|
Type: sch.StorageTypeURL,
|
||||||
ClusterID: uploadersdk.ClusterID(info.Cluster),
|
ClusterID: uploadersdk.ClusterID(info.Cluster),
|
||||||
JCSUploadInfo: cdsapi.ObjectUploadInfo{
|
JCSUploadInfo: cdsapi.ObjectUploadInfo{
|
||||||
UserID: s.userID,
|
UserID: s.userID,
|
||||||
|
|
|
||||||
|
|
@ -111,8 +111,8 @@ func (s *PCMJobCreate) do(rtx jobmgr.JobStateRunContext) (*sch.CreateJobResp, er
|
||||||
}
|
}
|
||||||
|
|
||||||
req := sch.CreateJobReq{
|
req := sch.CreateJobReq{
|
||||||
//Name: s.jobInfo.Name,
|
Name: s.jobInfo.Name,
|
||||||
//Description: s.jobInfo.Description,
|
Description: s.jobInfo.Description,
|
||||||
DataDistribute: dataDistribute,
|
DataDistribute: dataDistribute,
|
||||||
JobResources: s.jobInfo.JobResources,
|
JobResources: s.jobInfo.JobResources,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"gitlink.org.cn/JointCloud/pcm-hub/aikit/common/algorithm"
|
||||||
|
"gitlink.org.cn/JointCloud/pcm-hub/aikit/common/dataset"
|
||||||
|
"gitlink.org.cn/JointCloud/pcm-hub/aikit/common/model"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
||||||
|
|
@ -432,33 +435,90 @@ func (svc *JobSetService) DataBinding(id uploadersdk.DataID, userID cdssdk.UserI
|
||||||
|
|
||||||
switch bd := info.(type) {
|
switch bd := info.(type) {
|
||||||
case *sch.DatasetBinding:
|
case *sch.DatasetBinding:
|
||||||
jsonData, err := json.Marshal(bd)
|
packages, err := svc.db.UploadData().GetByPackageID(svc.db.DefCtx(), bd.PackageIDs, []int64{-2})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(jsonData))
|
if len(packages) == 0 {
|
||||||
|
return fmt.Errorf("no package found")
|
||||||
|
}
|
||||||
|
|
||||||
|
filePath := sch.Split + sch.DATASET + sch.Split + packages[0].PackageName
|
||||||
|
ds := &dataset.Dataset{Name: bd.Name, Description: bd.Description, FilePath: filePath, Category: dataset.CommonValue(bd.Category)}
|
||||||
|
resp, err := svc.hubClient.BindDataset("ModelArts", ds)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
jsonData, err := json.Marshal(resp.Data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
content, err := json.Marshal(bd)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(content), string(jsonData))
|
||||||
packageIDs = bd.PackageIDs
|
packageIDs = bd.PackageIDs
|
||||||
case *sch.CodeBinding:
|
case *sch.CodeBinding:
|
||||||
isCode = true
|
packages, err := svc.db.UploadData().GetByPackageID(svc.db.DefCtx(), []cdssdk.PackageID{bd.PackageID}, []int64{-2})
|
||||||
jsonData, err := json.Marshal(bd)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(jsonData))
|
if len(packages) == 0 {
|
||||||
|
return fmt.Errorf("no package found")
|
||||||
|
}
|
||||||
|
|
||||||
|
filePath := sch.Split + sch.CODE + sch.Split + packages[0].PackageName
|
||||||
|
code := &algorithm.Algorithm{Name: bd.Name, Description: bd.Description, Engine: "TensorFlow", CodeDir: filePath, BootFile: filePath + sch.Split + bd.FilePath, Branch: "main"}
|
||||||
|
resp, err := svc.hubClient.BindAlgorithm("ModelArts", code)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
jsonData, err := json.Marshal(resp.Data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
isCode = true
|
||||||
|
content, err := json.Marshal(bd)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(content), string(jsonData))
|
||||||
packageIDs = []cdssdk.PackageID{bd.PackageID}
|
packageIDs = []cdssdk.PackageID{bd.PackageID}
|
||||||
case *sch.ImageBinding:
|
case *sch.ImageBinding:
|
||||||
jsonData, err := json.Marshal(bd)
|
content, err := json.Marshal(bd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(jsonData))
|
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(content), "")
|
||||||
packageIDs = bd.PackageIDs
|
packageIDs = bd.PackageIDs
|
||||||
case *sch.ModelBinding:
|
case *sch.ModelBinding:
|
||||||
jsonData, err := json.Marshal(bd)
|
packages, err := svc.db.UploadData().GetByPackageID(svc.db.DefCtx(), bd.PackageIDs, []int64{-2})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(jsonData))
|
if len(packages) == 0 {
|
||||||
|
return fmt.Errorf("no package found")
|
||||||
|
}
|
||||||
|
|
||||||
|
filePath := sch.Split + sch.MODEL + sch.Split + packages[0].PackageName
|
||||||
|
md := &model.Model{Name: bd.Name, Description: bd.Description, Type: bd.Category, Version: bd.Version, Engine: model.CommonValue(bd.Env), FilePath: filePath}
|
||||||
|
resp, err := svc.hubClient.BindModel("ModelArts", md)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
jsonData, err := json.Marshal(resp.Data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
content, err := json.Marshal(bd)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
bindingData = getBindingData(id, userID, bd.Type, bd.Name, string(content), string(jsonData))
|
||||||
packageIDs = bd.PackageIDs
|
packageIDs = bd.PackageIDs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -492,13 +552,14 @@ func (svc *JobSetService) DataBinding(id uploadersdk.DataID, userID cdssdk.UserI
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getBindingData(id uploadersdk.DataID, userID cdssdk.UserID, dataType string, name string, content string) uploadersdk.Binding {
|
func getBindingData(id uploadersdk.DataID, userID cdssdk.UserID, dataType string, name string, content string, jsonData string) uploadersdk.Binding {
|
||||||
bindingData := uploadersdk.Binding{
|
bindingData := uploadersdk.Binding{
|
||||||
ID: id,
|
ID: id,
|
||||||
Name: name,
|
Name: name,
|
||||||
DataType: dataType,
|
DataType: dataType,
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
Content: content,
|
Content: content,
|
||||||
|
JsonData: jsonData,
|
||||||
}
|
}
|
||||||
return bindingData
|
return bindingData
|
||||||
}
|
}
|
||||||
|
|
@ -705,12 +766,17 @@ func (svc *JobSetService) CreatePackage(userID cdssdk.UserID, name string, dataT
|
||||||
}
|
}
|
||||||
defer schglb.CloudreamStoragePool.Release(cdsCli)
|
defer schglb.CloudreamStoragePool.Release(cdsCli)
|
||||||
|
|
||||||
bucketID := cdssdk.BucketID(1)
|
bucket, err := svc.db.Access().GetBucketByUserID(svc.db.DefCtx(), userID, dataType)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get bucket: %w", err)
|
||||||
|
} else if bucket == nil {
|
||||||
|
return fmt.Errorf("bucket not found")
|
||||||
|
}
|
||||||
|
|
||||||
// 创建package
|
// 创建package
|
||||||
newPackage, err := cdsCli.Package().Create(cdsapi.PackageCreate{
|
newPackage, err := cdsCli.Package().Create(cdsapi.PackageCreate{
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
BucketID: bucketID,
|
BucketID: bucket.ID,
|
||||||
Name: name,
|
Name: name,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -721,7 +787,7 @@ func (svc *JobSetService) CreatePackage(userID cdssdk.UserID, name string, dataT
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
PackageID: newPackage.Package.PackageID,
|
PackageID: newPackage.Package.PackageID,
|
||||||
PackageName: name,
|
PackageName: name,
|
||||||
BucketID: bucketID,
|
BucketID: bucket.ID,
|
||||||
DataType: dataType,
|
DataType: dataType,
|
||||||
UploadPriority: uploadPriority,
|
UploadPriority: uploadPriority,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package services
|
package services
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
hub "gitlink.org.cn/JointCloud/pcm-hub/client"
|
||||||
"gitlink.org.cn/cloudream/scheduler/common/pkgs/db"
|
"gitlink.org.cn/cloudream/scheduler/common/pkgs/db"
|
||||||
"gitlink.org.cn/cloudream/scheduler/common/pkgs/prescheduler2"
|
"gitlink.org.cn/cloudream/scheduler/common/pkgs/prescheduler2"
|
||||||
"gitlink.org.cn/cloudream/scheduler/schedulerMiddleware/internal/manager/jobmgr"
|
"gitlink.org.cn/cloudream/scheduler/schedulerMiddleware/internal/manager/jobmgr"
|
||||||
|
|
@ -10,12 +11,14 @@ type Service struct {
|
||||||
preScheduler prescheduler2.PreScheduler
|
preScheduler prescheduler2.PreScheduler
|
||||||
jobMgr *jobmgr.Manager
|
jobMgr *jobmgr.Manager
|
||||||
db *db.DB
|
db *db.DB
|
||||||
|
hubClient *hub.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewService(preScheduler prescheduler2.PreScheduler, jobMgr *jobmgr.Manager, db *db.DB) (*Service, error) {
|
func NewService(preScheduler prescheduler2.PreScheduler, jobMgr *jobmgr.Manager, db *db.DB, hubClient *hub.Client) (*Service, error) {
|
||||||
return &Service{
|
return &Service{
|
||||||
preScheduler: preScheduler,
|
preScheduler: preScheduler,
|
||||||
jobMgr: jobMgr,
|
jobMgr: jobMgr,
|
||||||
db: db,
|
db: db,
|
||||||
|
hubClient: hubClient,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,12 @@
|
||||||
package services
|
package services
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler"
|
sch "gitlink.org.cn/cloudream/common/sdks/pcmscheduler"
|
||||||
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
|
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
|
||||||
|
"gitlink.org.cn/cloudream/common/sdks/storage/cdsapi"
|
||||||
|
schglb "gitlink.org.cn/cloudream/scheduler/common/globals"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -24,13 +28,51 @@ func (svc *UserService) QueryUser(id string) (cdssdk.UserID, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (svc *UserService) AddUser(id string, userName string) error {
|
func (svc *UserService) AddUser(id string, userName string) error {
|
||||||
|
cdsCli, err := schglb.CloudreamStoragePool.Acquire()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("new scheduler client: %w", err)
|
||||||
|
}
|
||||||
|
defer schglb.CloudreamStoragePool.Release(cdsCli)
|
||||||
|
|
||||||
|
// 注册用户
|
||||||
|
createReq := &cdsapi.UserCreate{
|
||||||
|
Name: userName,
|
||||||
|
}
|
||||||
|
createResp, err := cdsCli.UserCreate(createReq)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create user: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
user := sch.User{
|
user := sch.User{
|
||||||
|
ID: int64(createResp.User.UserID),
|
||||||
SsoID: id,
|
SsoID: id,
|
||||||
UserName: userName,
|
UserName: userName,
|
||||||
Created: time.Now(),
|
Created: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := svc.db.Access().AddUser(svc.db.DefCtx(), user)
|
// 根据不同的数据类型，创建不同的桶
|
||||||
|
var buckets []sch.Bucket
|
||||||
|
dataTypes := []string{sch.CODE, sch.DATASET, sch.MODEL, sch.IMAGE}
|
||||||
|
for _, dataType := range dataTypes {
|
||||||
|
bucketName := strconv.FormatInt(int64(createResp.User.UserID), 10) + "_" + dataType
|
||||||
|
bucketCreateReq := cdsapi.BucketCreate{
|
||||||
|
Name: bucketName,
|
||||||
|
UserID: createResp.User.UserID,
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketCreateResp, err := cdsCli.Bucket().Create(bucketCreateReq)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
buckets = append(buckets, sch.Bucket{
|
||||||
|
ID: bucketCreateResp.Bucket.BucketID,
|
||||||
|
UserID: createResp.User.UserID,
|
||||||
|
DataType: dataType,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
err = svc.db.Access().AddUser(svc.db.DefCtx(), user, buckets)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,8 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
pcmHubClient "gitlink.org.cn/JointCloud/pcm-hub/client"
|
||||||
|
pcmHubConfig "gitlink.org.cn/JointCloud/pcm-hub/config"
|
||||||
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
||||||
schglb "gitlink.org.cn/cloudream/scheduler/common/globals"
|
schglb "gitlink.org.cn/cloudream/scheduler/common/globals"
|
||||||
"gitlink.org.cn/cloudream/scheduler/common/pkgs/db"
|
"gitlink.org.cn/cloudream/scheduler/common/pkgs/db"
|
||||||
|
|
@ -42,7 +44,15 @@ func main() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatalf("new job manager failed, err: %s", err.Error())
|
logger.Fatalf("new job manager failed, err: %s", err.Error())
|
||||||
}
|
}
|
||||||
svc, err := services.NewService(preSchr, jobMgr, dbSvc)
|
|
||||||
|
hubConfig := &pcmHubConfig.Config{
|
||||||
|
Platforms: config.Cfg().PCMHub,
|
||||||
|
}
|
||||||
|
hubClient, err := pcmHubClient.NewClient(hubConfig)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatalf("new pcm hub client failed, err: %s", err.Error())
|
||||||
|
}
|
||||||
|
svc, err := services.NewService(preSchr, jobMgr, dbSvc, hubClient)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Fatalf("new service failed, err: %s", err.Error())
|
logger.Fatalf("new service failed, err: %s", err.Error())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue