136 lines
6.3 KiB
Go
136 lines
6.3 KiB
Go
package utils
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"gitlink.org.cn/cloudream/common/pkgs/logger"
|
|
schglb "gitlink.org.cn/cloudream/scheduler/common/globals"
|
|
schmod "gitlink.org.cn/cloudream/scheduler/common/models"
|
|
"math/rand"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
schsdk "gitlink.org.cn/cloudream/common/sdks/scheduler"
|
|
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
|
|
)
|
|
|
|
func MakeJobOutputPath(userID cdssdk.UserID, jobID schsdk.JobID) string {
|
|
path := "jobs/" + strconv.FormatInt(int64(userID), 10) + "/" + string(jobID) + "/output"
|
|
//return filepath.Join("jobs", strconv.FormatInt(int64(userID), 10), string(jobID), "output")
|
|
return path
|
|
}
|
|
|
|
func MakeResourcePackageName(jobID schsdk.JobID) string {
|
|
return fmt.Sprintf("%s@%s", string(jobID), time.Now().Format("2006-01-02 15:04:05"))
|
|
}
|
|
|
|
func GenerateRandomID() string {
|
|
currentTime := time.Now().UnixNano() / int64(time.Millisecond)
|
|
rand.Seed(currentTime)
|
|
randomNum := rand.Intn(1000) // 0 到 999 之间的随机整数
|
|
idBase := fmt.Sprintf("%d%03d", currentTime, randomNum)
|
|
hasher := sha256.New()
|
|
hasher.Write([]byte(idBase))
|
|
hashBytes := hasher.Sum(nil)
|
|
hashedID := hex.EncodeToString(hashBytes)
|
|
return hashedID
|
|
}
|
|
|
|
func SplitCommands(command string) []string {
|
|
var commands []string
|
|
command = strings.Replace(command, "\\r\\n", "\\r//n", -1)
|
|
cmdArr := strings.Split(command, "\\n")
|
|
for i := 0; i < len(cmdArr); i++ {
|
|
if strings.Contains(cmdArr[i], "\\r//n") {
|
|
cmdArr[i] = strings.Replace(cmdArr[i], "\\r//n", "\\r\\n", -1)
|
|
}
|
|
commands = append(commands, strings.Trim(cmdArr[i], " "))
|
|
}
|
|
|
|
return commands
|
|
}
|
|
|
|
func ConvertEnvsToCommand(envs []schsdk.KVPair) []string {
|
|
var commands []string
|
|
|
|
for i := 0; i < len(envs); i++ {
|
|
value := strings.Replace(envs[i].Value, "\\", "/", -1)
|
|
commandContent := "sed -i '/@key@/d' ~/.bashrc && echo 'export @key@=@value@' >> ~/.bashrc"
|
|
commandContent = strings.Replace(commandContent, "@key@", envs[i].Key, -1)
|
|
commandContent = strings.Replace(commandContent, "@value@", value, -1)
|
|
logger.Info("env: " + commandContent)
|
|
commands = append(commands, commandContent)
|
|
}
|
|
commandContent := "sudo source ~/.bashrc"
|
|
commands = append(commands, commandContent)
|
|
|
|
return commands
|
|
}
|
|
|
|
func GetRcloneCommands(storage schmod.ObjectStorage, userID cdssdk.UserID, mountDir string) []string {
|
|
var commands []string
|
|
|
|
// 下载Rclone
|
|
commandContent := "yum install -y fuse3"
|
|
commands = append(commands, commandContent)
|
|
commandContent = "cd /opt && downloadCode='import requests;response=requests.get(\"@url@\",stream=True);response.raise_for_status();boundary=response.headers.get(\"Content-Type\").split(\"boundary=\")[-1].encode();content=response.content;body=[part.split(b\"\\r\\n\\r\\n\",1)[1].rsplit(b\"\\r\\n--\",1)[0] for part in content.split(b\"--\"+boundary+b\"\\r\\n\") if b\"filename=\" in part][0];open(\"@filename@\",\"wb\").write(body);print(\"success\")' && rclone=\"$cds_url/object/download?userID=$userID&objectID=$rcloneID\" && python3 -c \"$(echo \"$downloadCode\" | sed -e \"s|@url@|$(printf '%s' \"$rclone\" | sed 's/[&/\\]/\\\\&/g')|\" -e \"s|@filename@|rclone|\")\" && chmod +x rclone"
|
|
commandContent = strings.Replace(commandContent, "$cds_url", schglb.CloudreamStorageConfig.URL, -1)
|
|
commandContent = strings.Replace(commandContent, "$rcloneID", schglb.CDSRclone.CDSRcloneID, -1)
|
|
commandContent = strings.Replace(commandContent, "$userID", strconv.FormatInt(int64(userID), 10), -1)
|
|
commands = append(commands, commandContent)
|
|
|
|
// 生成Rclone配置文件
|
|
commandContent = "echo -e '[@tagName@] \n type = s3 \n provider = @provider@ \n access_key_id = @ak@ \n secret_access_key = @sk@ \n endpoint = @endpoint@ \n storage_class = STANDARD' > /opt/rclone.conf"
|
|
tagName := storage.Bucket + "_" + storage.AK
|
|
commandContent = strings.Replace(commandContent, "@tagName@", tagName, -1)
|
|
commandContent = strings.Replace(commandContent, "@provider@", storage.Manufacturer, -1)
|
|
commandContent = strings.Replace(commandContent, "@ak@", storage.AK, -1)
|
|
commandContent = strings.Replace(commandContent, "@sk@", storage.SK, -1)
|
|
commandContent = strings.Replace(commandContent, "@endpoint@", storage.Endpoint, -1)
|
|
commands = append(commands, commandContent)
|
|
|
|
//umountCommand := "umount -l /mnt/oss"
|
|
//commands = append(commands, umountCommand)
|
|
// 挂载Rclone
|
|
commandContent = "mkdir -p @mountDir@ && cd /opt && nohup ./rclone mount @tagName@:@bucket@ @mountDir@ --vfs-cache-mode full --vfs-read-wait 0 --vfs-read-chunk-size 128M --cache-db-purge -vv > rcloneMount.log 2>&1 &"
|
|
commandContent = strings.Replace(commandContent, "@tagName@", tagName, -1)
|
|
commandContent = strings.Replace(commandContent, "@bucket@", storage.Bucket, -1)
|
|
commandContent = strings.Replace(commandContent, "@mountDir@", mountDir, -1)
|
|
commands = append(commands, commandContent)
|
|
|
|
commandContent = "cd /opt && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda3 && eval \"$($HOME/miniconda3/bin/conda shell.bash hook)\" && conda create -n myenv python=3.10 -y"
|
|
commands = append(commands, commandContent)
|
|
|
|
return commands
|
|
}
|
|
|
|
func RemountRclone(storage schmod.ObjectStorage, userID cdssdk.UserID, mountDir string) string {
|
|
umountCommand := "umount -l /mnt/oss"
|
|
// 挂载Rclone
|
|
commandContent := "mkdir -p @mountDir@ && cd /opt && nohup ./rclone mount @tagName@:@bucket@ @mountDir@ --vfs-cache-mode full --vfs-read-wait 0 --vfs-read-chunk-size 128M --cache-db-purge -vv > rcloneMount.log 2>&1 &"
|
|
tagName := storage.Bucket + "_" + storage.AK
|
|
commandContent = strings.Replace(commandContent, "@tagName@", tagName, -1)
|
|
commandContent = strings.Replace(commandContent, "@bucket@", storage.Bucket, -1)
|
|
commandContent = strings.Replace(commandContent, "@mountDir@", mountDir, -1)
|
|
return umountCommand + " \n " + commandContent
|
|
}
|
|
|
|
func HandleCommand(startScript string) string {
|
|
|
|
startScript = strings.Replace(startScript, "//", "/", -1)
|
|
commandContent := "sudo sh @startScript@ > /opt/@startLog@.log"
|
|
commandContent = strings.Replace(commandContent, "@startScript@", startScript, -1)
|
|
arr := strings.Split(startScript, "/")
|
|
commandContent = strings.Replace(commandContent, "@startLog@", arr[len(arr)-1], -1)
|
|
|
|
return commandContent
|
|
}
|
|
|
|
func HandlePath(path string) string {
|
|
path = strings.ReplaceAll(path, "\\", "/")
|
|
path = strings.ReplaceAll(path, "//", "/")
|
|
return path
|
|
}
|