Merge branch 'ai-ui-tars' into 'v5'
新增 LLM 能力，集成 UI-Tars See merge request iesqa/httprunner!77
This commit is contained in:
李隆
commit
379f0de4ea
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
.env
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -16,10 +16,12 @@ bump: ## bump hrp version, e.g. make bump version=4.0.0
|
|||
.PHONY: build
|
||||
build: ## build hrp cli tool
|
||||
@echo "[info] build hrp cli tool"
|
||||
go mod tidy
|
||||
go build -ldflags "\
|
||||
-s -w \
|
||||
-X 'github.com/httprunner/httprunner/v5/internal/version.GitCommit=$(shell git rev-parse HEAD)' \
|
||||
-X 'github.com/httprunner/httprunner/v5/internal/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)' \
|
||||
-X 'github.com/httprunner/httprunner/v5/internal/version.GitAuthor=$(shell git log -1 --pretty=format:%an)' \
|
||||
-X 'github.com/httprunner/httprunner/v5/internal/version.BuildTime=$(shell date "+%y%m%d%H%M")'" \
|
||||
-o output/hrp ./cmd/cli
|
||||
./output/hrp -v
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ var (
|
|||
CVResponseError = errors.New("CV parse response error") // 83
|
||||
CVResultNotFoundError = errors.New("CV result not found") // 84
|
||||
|
||||
StateUnknowError = errors.New("detect state failed") // 85
|
||||
LLMEnvMissedError = errors.New("LLM env missed error") // 85
|
||||
StateUnknowError = errors.New("detect state failed") // 89
|
||||
)
|
||||
|
||||
// trackings related: [90, 100)
|
||||
|
|
@ -199,7 +200,8 @@ var errorsMap = map[error]int{
|
|||
CVServiceConnectionError: 82,
|
||||
CVResponseError: 83,
|
||||
CVResultNotFoundError: 84,
|
||||
StateUnknowError: 85,
|
||||
LLMEnvMissedError: 85,
|
||||
StateUnknowError: 89,
|
||||
|
||||
// trackings related
|
||||
TrackingGetError: 90,
|
||||
|
|
|
|||
34
go.mod
34
go.mod
|
|
@ -5,6 +5,8 @@ go 1.22.0
|
|||
require (
|
||||
github.com/Masterminds/semver v1.5.0
|
||||
github.com/andybalholm/brotli v1.0.4
|
||||
github.com/cloudwego/eino v0.3.16
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c
|
||||
github.com/danielpaulus/go-ios v1.0.161
|
||||
github.com/denisbrodbeck/machineid v1.0.1
|
||||
github.com/fatih/color v1.16.0
|
||||
|
|
@ -16,29 +18,33 @@ require (
|
|||
github.com/httprunner/funplugin v0.5.5
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/jmespath/go-jmespath v0.4.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/maja42/goval v1.2.1
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/zerolog v1.30.0
|
||||
github.com/rs/zerolog v1.33.0
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/spf13/cobra v1.5.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
golang.org/x/net v0.26.0
|
||||
golang.org/x/text v0.16.0
|
||||
golang.org/x/text v0.21.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bytedance/sonic v1.11.6 // indirect
|
||||
github.com/bytedance/sonic/loader v0.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.12.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.0 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/getkin/kin-openapi v0.118.0 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-errors/errors v1.4.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
|
|
@ -53,11 +59,13 @@ require (
|
|||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/goph/emperror v0.17.2 // indirect
|
||||
github.com/grandcat/zeroconf v1.0.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.5.0 // indirect
|
||||
github.com/hashicorp/go-plugin v1.4.10 // indirect
|
||||
github.com/hashicorp/yamux v0.1.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/invopop/yaml v0.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
|
|
@ -69,29 +77,35 @@ require (
|
|||
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/nikolalohinski/gonja v1.5.3 // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/perimeterx/marshmallow v1.1.4 // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
|
||||
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sashabaranov/go-openai v1.32.5 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/yargevad/filepathx v1.0.0 // indirect
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
|
||||
go.uber.org/mock v0.3.0 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.24.0 // indirect
|
||||
go.uber.org/mock v0.4.0 // indirect
|
||||
golang.org/x/arch v0.11.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
|
||||
golang.org/x/mod v0.17.0 // indirect
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/sys v0.21.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
|
||||
|
|
|
|||
123
go.sum
123
go.sum
|
|
@ -1,18 +1,33 @@
|
|||
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
|
||||
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
|
||||
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
|
||||
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
|
||||
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
|
||||
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
|
||||
github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
|
||||
github.com/bytedance/mockey v1.2.13 h1:jokWZAm/pUEbD939Rhznz615MKUCZNuvCFQlJ2+ntoo=
|
||||
github.com/bytedance/mockey v1.2.13/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY=
|
||||
github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg=
|
||||
github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM=
|
||||
github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
|
||||
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/eino v0.3.16 h1:ASN8zISyoEdjEsPnIw5GazSHtbNY97NDthQ2B69yiZw=
|
||||
github.com/cloudwego/eino v0.3.16/go.mod h1:+kmJimGEcKuSI6OKhet7kBedkm1WUZS3H1QRazxgWUo=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c h1:04WQpGikdQv6fh5wzMYSQhO0SJraV8+xcb9VQ00+HX4=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c/go.mod h1:YGP4q3uspj5qhkv3CnvlEPSo0YGeWpvkkTUHHpLExas=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 h1:fgvkmTqAalDfjdy3b6Ur2mh/KEwB9L2uvqS4MFgTOqc=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955/go.mod h1:6CThw1XQx/ASXNt31yuvp0X4Yp4GprknQuIvP9VKDpw=
|
||||
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
|
|
@ -26,6 +41,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
|
||||
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0=
|
||||
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
|
|
@ -33,14 +50,20 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
|||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM=
|
||||
github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc=
|
||||
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||
github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
|
||||
github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
|
||||
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
|
|
@ -66,9 +89,13 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
|
|||
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
|
||||
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
|
|
@ -84,6 +111,11 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
|
|||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18=
|
||||
github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
|
||||
|
|
@ -94,12 +126,15 @@ github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQ
|
|||
github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0=
|
||||
github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
|
||||
github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/httprunner/funplugin v0.5.5 h1:VU1a6kj1AsJ/ucIhhI5NLHXOP4xnW2JGgk50vBV3Zis=
|
||||
github.com/httprunner/funplugin v0.5.5/go.mod h1:YZzBBSOSdLZEpHZz0P2E5SOQ+o1+Fbn30oWS4RGHBz0=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
|
||||
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
|
||||
github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
|
||||
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
|
||||
|
|
@ -109,14 +144,20 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
|
|||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
|
|
@ -140,8 +181,11 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
|||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
|
||||
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
|
||||
|
|
@ -154,20 +198,30 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c=
|
||||
github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4=
|
||||
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw=
|
||||
github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
|
@ -179,15 +233,25 @@ github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55/go.mod h1:PeN7k
|
|||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
|
||||
github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
|
||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sashabaranov/go-openai v1.32.5 h1:/eNVa8KzlE7mJdKPZDj6886MUzZQjoVHyn0sLvIt5qA=
|
||||
github.com/sashabaranov/go-openai v1.32.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI=
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg=
|
||||
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
|
||||
github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
|
||||
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
|
||||
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
|
||||
github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
|
||||
|
|
@ -195,10 +259,12 @@ github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJ
|
|||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
|
@ -207,31 +273,39 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
|
|||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 h1:aeN+ghOV0b2VCmKKO3gqnDQ8mLbpABZgRR2FVYx4ouI=
|
||||
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9/go.mod h1:roo6cZ/uqpwKMuvPG0YmzI5+AmUiMWfjCBZpGXqbTxE=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
|
||||
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
|
||||
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
|
||||
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
|
||||
go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
|
||||
go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
|
||||
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4=
|
||||
golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
|
@ -239,9 +313,12 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -258,13 +335,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
@ -288,13 +368,17 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
|||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5 h1:DOUDfNS+CFMM46k18FRF5k/0yz5NhZYMiUQxf4xglIU=
|
||||
|
|
@ -302,6 +386,5 @@ gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5/go.mod h1:NQHVAzMwvZ+Qe3ElS
|
|||
howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
|
||||
howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE=
|
||||
software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ=
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
v5.0.0-beta-2503251404
|
||||
v5.0.0-beta-2503311103
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ var (
|
|||
GitCommit = "unknown"
|
||||
GitBranch = "unknown"
|
||||
BuildTime = "unknown"
|
||||
GitAuthor = "unknown"
|
||||
)
|
||||
|
||||
func GetVersionInfo() string {
|
||||
|
|
@ -26,6 +27,6 @@ func GetVersionInfo() string {
|
|||
if len(commitID) > 8 {
|
||||
commitID = commitID[:8]
|
||||
}
|
||||
return fmt.Sprintf("%s (branch=%s, commit=%s, build=%s)",
|
||||
VERSION, GitBranch, commitID, BuildTime)
|
||||
return fmt.Sprintf("%s (branch=%s, commit=%s, author=%s, build=%s)",
|
||||
VERSION, GitBranch, commitID, GitAuthor, BuildTime)
|
||||
}
|
||||
|
|
|
|||
27
step_ui.go
27
step_ui.go
|
|
@ -164,6 +164,18 @@ func (s *StepMobile) TapByUITypes(opts ...option.ActionOption) *StepMobile {
|
|||
return s
|
||||
}
|
||||
|
||||
// AIAction do actions with VLM
|
||||
func (s *StepMobile) AIAction(prompt string, opts ...option.ActionOption) *StepMobile {
|
||||
action := uixt.MobileAction{
|
||||
Method: uixt.ACTION_AIAction,
|
||||
Params: prompt,
|
||||
Options: option.NewActionOptions(opts...),
|
||||
}
|
||||
|
||||
s.obj().Actions = append(s.obj().Actions, action)
|
||||
return s
|
||||
}
|
||||
|
||||
// DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates
|
||||
func (s *StepMobile) DoubleTapXY(x, y float64, opts ...option.ActionOption) *StepMobile {
|
||||
s.obj().Actions = append(s.obj().Actions, uixt.MobileAction{
|
||||
|
|
@ -527,6 +539,21 @@ func (s *StepMobileUIValidation) AssertImageNotExists(expectedImagePath string,
|
|||
return s
|
||||
}
|
||||
|
||||
func (s *StepMobileUIValidation) AssertAI(prompt string, msg ...string) *StepMobileUIValidation {
|
||||
v := Validator{
|
||||
Check: uixt.SelectorAI,
|
||||
Assert: uixt.AssertionAI,
|
||||
Expect: prompt,
|
||||
}
|
||||
if len(msg) > 0 {
|
||||
v.Message = msg[0]
|
||||
} else {
|
||||
v.Message = fmt.Sprintf("assert ai prompt [%s] failed", prompt)
|
||||
}
|
||||
s.Validators = append(s.Validators, v)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg ...string) *StepMobileUIValidation {
|
||||
v := Validator{
|
||||
Check: uixt.SelectorForegroundApp,
|
||||
|
|
|
|||
|
|
@ -80,3 +80,21 @@ func TestAndroidAction(t *testing.T) {
|
|||
err := hrp.NewRunner(t).Run(testCase)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestAIAction(t *testing.T) {
|
||||
testCase := &hrp.TestCase{
|
||||
Config: hrp.NewConfig("run ui action with ai"),
|
||||
TestSteps: []hrp.IStep{
|
||||
hrp.NewStep("launch settings").
|
||||
Android().AIAction("进入手机系统设置").
|
||||
Validate().
|
||||
AssertAI("当前位于手机设置页面"),
|
||||
hrp.NewStep("turn on fly mode").
|
||||
Android().AIAction("开启飞行模式").
|
||||
Validate().
|
||||
AssertAI("飞行模式已开启"),
|
||||
},
|
||||
}
|
||||
err := hrp.NewRunner(t).Run(testCase)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
|
@ -46,11 +47,17 @@ func WithCVService(service CVServiceType) AIServiceOption {
|
|||
type LLMServiceType string
|
||||
|
||||
const (
|
||||
LLMServiceTypeUITARS LLMServiceType = "ui-tars"
|
||||
LLMServiceTypeGPT4o LLMServiceType = "gpt-4o"
|
||||
LLMServiceTypeDeepSeekV3 LLMServiceType = "deepseek-v3"
|
||||
)
|
||||
|
||||
func WithLLMService(service LLMServiceType) AIServiceOption {
|
||||
if err := checkEnvLLM(); err != nil {
|
||||
log.Error().Err(err).Msg("check LLM env failed")
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
|
||||
return func(opts *AIServices) {
|
||||
if service == LLMServiceTypeGPT4o {
|
||||
var err error
|
||||
|
|
@ -60,5 +67,12 @@ func WithLLMService(service LLMServiceType) AIServiceOption {
|
|||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
}
|
||||
if service == LLMServiceTypeUITARS {
|
||||
var err error
|
||||
opts.ILLMService, err = NewPlanner(context.Background())
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init ui-tars llm service failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
package ai
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOption(t *testing.T) {
|
||||
options := NewAIService(
|
||||
WithCVService(CVServiceTypeOpenCV),
|
||||
WithLLMService(LLMServiceTypeDeepSeekV3),
|
||||
WithLLMService(LLMServiceTypeUITARS),
|
||||
)
|
||||
t.Log(options)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ type APIResponseImage struct {
|
|||
}
|
||||
|
||||
func NewVEDEMImageService() (*vedemCVService, error) {
|
||||
if err := checkEnv(); err != nil {
|
||||
if err := checkEnvCV(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &vedemCVService{}, nil
|
||||
|
|
@ -230,7 +230,7 @@ func (s *vedemCVService) ReadFromBuffer(imageBuf *bytes.Buffer, opts ...option.A
|
|||
return imageResult, nil
|
||||
}
|
||||
|
||||
func checkEnv() error {
|
||||
func checkEnvCV() error {
|
||||
vedemImageURL := os.Getenv("VEDEM_IMAGE_URL")
|
||||
if vedemImageURL == "" {
|
||||
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_URL missed")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
type OpenAIInitConfig struct {
|
||||
ReportURL string `json:"REPORT_SERVER_URL"`
|
||||
Headers map[string]string `json:"defaultHeaders"`
|
||||
}
|
||||
|
||||
const (
|
||||
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
|
||||
EnvOpenAIAPIKey = "OPENAI_API_KEY"
|
||||
EnvModelName = "LLM_MODEL_NAME"
|
||||
EnvOpenAIInitConfigJSON = "OPENAI_INIT_CONFIG_JSON"
|
||||
)
|
||||
|
||||
var once sync.Once
|
||||
|
||||
// loadEnv loads environment variables from .env file
|
||||
// it will search for .env file from current working directory upward recursively
|
||||
func loadEnv() {
|
||||
once.Do(func() {
|
||||
// get current working directory
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// locate .env file from current working directory upward recursively
|
||||
envPath := cwd
|
||||
for {
|
||||
envFile := filepath.Join(envPath, ".env")
|
||||
if _, err := os.Stat(envFile); err == nil {
|
||||
// found .env file
|
||||
// override existing env variables
|
||||
err = godotenv.Overload(envFile)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).
|
||||
Str("path", envFile).Msg("overload env file failed")
|
||||
}
|
||||
log.Info().Str("path", envFile).Msg("overload env success")
|
||||
return
|
||||
}
|
||||
|
||||
// reached root directory
|
||||
parent := filepath.Dir(envPath)
|
||||
if parent == envPath {
|
||||
log.Info().Msg("no .env file found from current directory to root")
|
||||
return
|
||||
}
|
||||
envPath = parent
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func checkEnvLLM() error {
|
||||
loadEnv()
|
||||
openaiBaseURL := os.Getenv("OPENAI_BASE_URL")
|
||||
if openaiBaseURL == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "OPENAI_BASE_URL missed")
|
||||
}
|
||||
log.Info().Str("OPENAI_BASE_URL", openaiBaseURL).Msg("get env")
|
||||
openaiAPIKey := os.Getenv("OPENAI_API_KEY")
|
||||
if openaiAPIKey == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "OPENAI_API_KEY missed")
|
||||
}
|
||||
log.Info().Str("OPENAI_API_KEY", maskAPIKey(openaiAPIKey)).Msg("get env")
|
||||
modelName := os.Getenv("LLM_MODEL_NAME")
|
||||
if modelName == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "LLM_MODEL_NAME missed")
|
||||
}
|
||||
log.Info().Str("LLM_MODEL_NAME", modelName).Msg("get env")
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetEnvConfig(key string) string {
|
||||
return os.Getenv(key)
|
||||
}
|
||||
|
||||
func GetEnvConfigInJSON(key string) (map[string]interface{}, error) {
|
||||
value := GetEnvConfig(key)
|
||||
if value == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(value), &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func GetEnvConfigInBool(key string) bool {
|
||||
value := GetEnvConfig(key)
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
boolValue, _ := strconv.ParseBool(value)
|
||||
return boolValue
|
||||
}
|
||||
|
||||
// GetEnvConfigOrDefault get env config or default value
|
||||
func GetEnvConfigOrDefault(key, defaultValue string) string {
|
||||
value := GetEnvConfig(key)
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func GetEnvConfigInInt(key string, defaultValue int) int {
|
||||
value := GetEnvConfig(key)
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
intValue, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return defaultValue
|
||||
}
|
||||
return intValue
|
||||
}
|
||||
|
||||
// CustomTransport is a custom RoundTripper that adds headers to every request
|
||||
type CustomTransport struct {
|
||||
Transport http.RoundTripper
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
// RoundTrip executes a single HTTP transaction and adds custom headers
|
||||
func (c *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
for key, value := range c.Headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
return c.Transport.RoundTrip(req)
|
||||
}
|
||||
|
||||
type OutputFormat struct {
|
||||
Thought string `json:"thought"`
|
||||
Action string `json:"action"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// GetModelConfig get OpenAI config
|
||||
func GetModelConfig() (*openai.ChatModelConfig, error) {
|
||||
loadEnv()
|
||||
envConfig := &OpenAIInitConfig{
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// read from JSON config first
|
||||
jsonStr := GetEnvConfig(EnvOpenAIInitConfigJSON)
|
||||
if jsonStr != "" {
|
||||
if err := json.Unmarshal([]byte(jsonStr), envConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
|
||||
// if err != nil {
|
||||
// log.Fatal().Err(err).Msg("NewSchemaRefForValue failed")
|
||||
// }
|
||||
|
||||
config := &openai.ChatModelConfig{
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: defaultTimeout,
|
||||
Transport: &CustomTransport{
|
||||
Transport: http.DefaultTransport,
|
||||
Headers: envConfig.Headers,
|
||||
},
|
||||
},
|
||||
// TODO: set structured response format
|
||||
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
|
||||
// ResponseFormat: &openai2.ChatCompletionResponseFormat{
|
||||
// Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
|
||||
// JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
|
||||
// Name: "thought_and_action",
|
||||
// Description: "data that describes planning thought and action",
|
||||
// Schema: outputFormatSchema.Value,
|
||||
// Strict: false,
|
||||
// },
|
||||
// },
|
||||
}
|
||||
|
||||
if baseURL := GetEnvConfig(EnvOpenAIBaseURL); baseURL != "" {
|
||||
config.BaseURL = baseURL
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvOpenAIBaseURL)
|
||||
}
|
||||
|
||||
if apiKey := GetEnvConfig(EnvOpenAIAPIKey); apiKey != "" {
|
||||
config.APIKey = apiKey
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvOpenAIAPIKey)
|
||||
}
|
||||
|
||||
if modelName := GetEnvConfig(EnvModelName); modelName != "" {
|
||||
config.Model = modelName
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvModelName)
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", config.Model).
|
||||
Str("baseURL", config.BaseURL).
|
||||
Str("apiKey", maskAPIKey(config.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// maskAPIKey masks the API key
|
||||
func maskAPIKey(key string) string {
|
||||
if len(key) <= 8 {
|
||||
return "******"
|
||||
}
|
||||
|
||||
return key[:4] + "******" + key[len(key)-4:]
|
||||
}
|
||||
|
|
@ -1,9 +1,12 @@
|
|||
package ai
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
|
||||
type ILLMService interface {
|
||||
Call(ctx context.Context, prompt string) (string, error)
|
||||
Call(opts *PlanningOptions) (*PlanningResult, error)
|
||||
}
|
||||
|
||||
func NewGPT4oLLMService() (*openaiLLMService, error) {
|
||||
|
|
@ -12,6 +15,46 @@ func NewGPT4oLLMService() (*openaiLLMService, error) {
|
|||
|
||||
type openaiLLMService struct{}
|
||||
|
||||
func (s openaiLLMService) Call(ctx context.Context, prompt string) (string, error) {
|
||||
return "", nil
|
||||
func (s openaiLLMService) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// PlanningOptions represents the input options for planning
|
||||
type PlanningOptions struct {
|
||||
UserInstruction string `json:"user_instruction"` // append to system prompt
|
||||
Message *schema.Message `json:"message"`
|
||||
Size types.Size `json:"size"`
|
||||
}
|
||||
|
||||
// PlanningResult represents the result of planning
|
||||
type PlanningResult struct {
|
||||
NextActions []ParsedAction `json:"actions"`
|
||||
ActionSummary string `json:"summary"`
|
||||
}
|
||||
|
||||
// VLMResponse represents the response from the Vision Language Model
|
||||
type VLMResponse struct {
|
||||
Actions []ParsedAction `json:"actions"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ParsedAction represents a parsed action from the VLM response
|
||||
type ParsedAction struct {
|
||||
ActionType ActionType `json:"actionType"`
|
||||
ActionInputs map[string]interface{} `json:"actionInputs"`
|
||||
Thought string `json:"thought"`
|
||||
}
|
||||
|
||||
type ActionType string
|
||||
|
||||
const (
|
||||
ActionTypeClick ActionType = "click"
|
||||
ActionTypeTap ActionType = "tap"
|
||||
ActionTypeDrag ActionType = "drag"
|
||||
ActionTypeSwipe ActionType = "swipe"
|
||||
ActionTypeWait ActionType = "wait"
|
||||
ActionTypeFinished ActionType = "finished"
|
||||
ActionTypeCallUser ActionType = "call_user"
|
||||
ActionTypeType ActionType = "type"
|
||||
ActionTypeScroll ActionType = "scroll"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,236 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// NewActionParser creates a new ActionParser instance
|
||||
func NewActionParser(factor float64) *ActionParser {
|
||||
return &ActionParser{
|
||||
Factor: factor,
|
||||
}
|
||||
}
|
||||
|
||||
// ActionParser parses VLM responses and converts them to structured actions
|
||||
type ActionParser struct {
|
||||
Factor float64 // TODO
|
||||
}
|
||||
|
||||
// Parse parses the prediction text and extracts actions
|
||||
func (p *ActionParser) Parse(predictionText string) ([]ParsedAction, error) {
|
||||
// try parsing JSON format, from VLM like openai/gpt-4o
|
||||
var jsonActions []ParsedAction
|
||||
jsonActions, jsonErr := p.parseJSON(predictionText)
|
||||
if jsonErr == nil {
|
||||
return jsonActions, nil
|
||||
}
|
||||
|
||||
// json parsing failed, try parsing Thought/Action format, from VLM like UI-TARS
|
||||
thoughtActions, thoughtErr := p.parseThoughtAction(predictionText)
|
||||
if thoughtErr == nil {
|
||||
return thoughtActions, nil
|
||||
}
|
||||
|
||||
return nil, errors.Wrap(thoughtErr, "parse planner response failed")
|
||||
}
|
||||
|
||||
// parseJSON tries to parse the response as JSON format
|
||||
func (p *ActionParser) parseJSON(predictionText string) ([]ParsedAction, error) {
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
|
||||
predictionText = strings.TrimPrefix(predictionText, "```json")
|
||||
predictionText = strings.TrimSuffix(predictionText, "```")
|
||||
}
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
|
||||
var response VLMResponse
|
||||
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
|
||||
if len(response.Actions) == 0 {
|
||||
return nil, errors.New("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// normalize actions
|
||||
var normalizedActions []ParsedAction
|
||||
for i := range response.Actions {
|
||||
// create a new variable, avoid implicit memory aliasing in for loop.
|
||||
action := response.Actions[i]
|
||||
if err := p.normalizeAction(&action); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to normalize action")
|
||||
}
|
||||
normalizedActions = append(normalizedActions, action)
|
||||
}
|
||||
|
||||
return normalizedActions, nil
|
||||
}
|
||||
|
||||
// parseThoughtAction parses the Thought/Action format response
|
||||
func (p *ActionParser) parseThoughtAction(predictionText string) ([]ParsedAction, error) {
|
||||
thoughtRegex := regexp.MustCompile(`(?is)Thought:(.+?)Action:`)
|
||||
actionRegex := regexp.MustCompile(`(?is)Action:(.+)`)
|
||||
|
||||
// extract Thought part
|
||||
thoughtMatch := thoughtRegex.FindStringSubmatch(predictionText)
|
||||
var thought string
|
||||
if len(thoughtMatch) > 1 {
|
||||
thought = strings.TrimSpace(thoughtMatch[1])
|
||||
}
|
||||
|
||||
// extract Action part, e.g. "click(start_box='(552,454)')"
|
||||
actionMatch := actionRegex.FindStringSubmatch(predictionText)
|
||||
if len(actionMatch) < 2 {
|
||||
return nil, errors.New("no action found in the response")
|
||||
}
|
||||
|
||||
actionText := strings.TrimSpace(actionMatch[1])
|
||||
|
||||
// parse action type and parameters
|
||||
return p.parseActionText(actionText, thought)
|
||||
}
|
||||
|
||||
// parseActionText parses the action text to extract the action type and parameters
|
||||
func (p *ActionParser) parseActionText(actionText, thought string) ([]ParsedAction, error) {
|
||||
// remove trailing comments
|
||||
if idx := strings.Index(actionText, "#"); idx > 0 {
|
||||
actionText = strings.TrimSpace(actionText[:idx])
|
||||
}
|
||||
|
||||
// supported action types and regexes
|
||||
actionRegexes := map[ActionType]*regexp.Regexp{
|
||||
"click": regexp.MustCompile(`click\(start_box='([^']+)'\)`),
|
||||
"left_double": regexp.MustCompile(`left_double\(start_box='([^']+)'\)`),
|
||||
"right_single": regexp.MustCompile(`right_single\(start_box='([^']+)'\)`),
|
||||
"drag": regexp.MustCompile(`drag\(start_box='([^']+)', end_box='([^']+)'\)`),
|
||||
"type": regexp.MustCompile(`type\(content='([^']+)'\)`),
|
||||
"scroll": regexp.MustCompile(`scroll\(start_box='([^']+)', direction='([^']+)'\)`),
|
||||
"wait": regexp.MustCompile(`wait\(\)`),
|
||||
"finished": regexp.MustCompile(`finished\(content='([^']+)'\)`),
|
||||
"call_user": regexp.MustCompile(`call_user\(\)`),
|
||||
}
|
||||
|
||||
parsedActions := make([]ParsedAction, 0)
|
||||
for actionType, regex := range actionRegexes {
|
||||
matches := regex.FindStringSubmatch(actionText)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var action ParsedAction
|
||||
action.ActionType = actionType
|
||||
action.ActionInputs = make(map[string]interface{})
|
||||
action.Thought = thought
|
||||
|
||||
// parse parameters based on action type
|
||||
switch actionType {
|
||||
case ActionTypeClick:
|
||||
if len(matches) > 1 {
|
||||
coord, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize point failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = coord
|
||||
}
|
||||
case ActionTypeDrag:
|
||||
if len(matches) > 2 {
|
||||
// handle start point
|
||||
startBox, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = startBox
|
||||
|
||||
// handle end point
|
||||
endBox, err := p.normalizeCoordinates(matches[2])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize endBox failed: %s", matches[2])
|
||||
}
|
||||
action.ActionInputs["endBox"] = endBox
|
||||
}
|
||||
case ActionTypeType:
|
||||
if len(matches) > 1 {
|
||||
action.ActionInputs["content"] = matches[1]
|
||||
}
|
||||
case ActionTypeScroll:
|
||||
if len(matches) > 2 {
|
||||
startBox, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = startBox
|
||||
action.ActionInputs["direction"] = matches[2]
|
||||
}
|
||||
case ActionTypeWait, ActionTypeFinished, ActionTypeCallUser:
|
||||
// 这些动作没有额外参数
|
||||
}
|
||||
|
||||
parsedActions = append(parsedActions, action)
|
||||
}
|
||||
|
||||
if len(parsedActions) == 0 {
|
||||
return nil, fmt.Errorf("no valid actions returned from VLM")
|
||||
}
|
||||
return parsedActions, nil
|
||||
}
|
||||
|
||||
// normalizeAction normalizes the coordinates in the action
|
||||
func (p *ActionParser) normalizeAction(action *ParsedAction) error {
|
||||
switch action.ActionType {
|
||||
case "click", "drag":
|
||||
// handle click and drag action coordinates
|
||||
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
|
||||
normalized, err := p.normalizeCoordinates(startBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize startBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["startBox"] = normalized
|
||||
}
|
||||
|
||||
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
|
||||
normalized, err := p.normalizeCoordinates(endBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize endBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["endBox"] = normalized
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeCoordinates normalizes the coordinates based on the factor
|
||||
func (p *ActionParser) normalizeCoordinates(coordStr string) (coords []float64, err error) {
|
||||
// check empty string
|
||||
if coordStr == "" {
|
||||
return nil, fmt.Errorf("empty coordinate string")
|
||||
}
|
||||
|
||||
if !strings.Contains(coordStr, ",") {
|
||||
return nil, fmt.Errorf("invalid coordinate string: %s", coordStr)
|
||||
}
|
||||
|
||||
// remove possible brackets and split coordinates
|
||||
coordStr = strings.Trim(coordStr, "[]() \t")
|
||||
|
||||
// try parsing JSON array
|
||||
jsonStr := coordStr
|
||||
if !strings.HasPrefix(jsonStr, "[") {
|
||||
jsonStr = "[" + coordStr + "]"
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(jsonStr), &coords)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse coordinate string: %w", err)
|
||||
}
|
||||
return coords, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,456 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
_ "image/jpeg"
|
||||
"image/png"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Error types
|
||||
var (
|
||||
ErrEmptyInstruction = fmt.Errorf("user instruction is empty")
|
||||
ErrNoConversationHistory = fmt.Errorf("conversation history is empty")
|
||||
ErrInvalidImageData = fmt.Errorf("invalid image data")
|
||||
)
|
||||
|
||||
func NewPlanner(ctx context.Context) (*Planner, error) {
|
||||
config, err := GetModelConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
|
||||
}
|
||||
model, err := openai.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
|
||||
}
|
||||
parser := NewActionParser(1000)
|
||||
return &Planner{
|
||||
ctx: ctx,
|
||||
config: config,
|
||||
model: model,
|
||||
parser: parser,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Planner struct {
|
||||
ctx context.Context
|
||||
model model.ChatModel
|
||||
config *openai.ChatModelConfig
|
||||
parser *ActionParser
|
||||
history []*schema.Message // conversation history
|
||||
}
|
||||
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validateInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate input parameters failed")
|
||||
}
|
||||
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
|
||||
p.history = []*schema.Message{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
p.appendConversationHistory(opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", p.config.Model).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request model service failed: %w", err)
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parse result failed")
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
p.appendConversationHistory(&schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateInput(opts *PlanningOptions) error {
|
||||
if opts.UserInstruction == "" {
|
||||
return ErrEmptyInstruction
|
||||
}
|
||||
|
||||
if opts.Message == nil {
|
||||
return ErrNoConversationHistory
|
||||
}
|
||||
|
||||
if opts.Message.Role == schema.User {
|
||||
// check MultiContent
|
||||
if len(opts.Message.MultiContent) > 0 {
|
||||
for _, content := range opts.Message.MultiContent {
|
||||
if content.Type == schema.ChatMessagePartTypeImageURL && content.ImageURL == nil {
|
||||
return ErrInvalidImageData
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func logRequest(messages []*schema.Message) {
|
||||
msgs := make([]*schema.Message, 0, len(messages))
|
||||
for _, message := range messages {
|
||||
msg := &schema.Message{
|
||||
Role: message.Role,
|
||||
}
|
||||
if message.Content != "" {
|
||||
msg.Content = message.Content
|
||||
} else if len(message.MultiContent) > 0 {
|
||||
for _, mc := range message.MultiContent {
|
||||
switch mc.Type {
|
||||
case schema.ChatMessagePartTypeImageURL:
|
||||
// Create a copy of the ImageURL to avoid modifying the original message
|
||||
imageURLCopy := *mc.ImageURL
|
||||
if strings.HasPrefix(imageURLCopy.URL, "data:image/") {
|
||||
imageURLCopy.URL = "<data:image/base64...>"
|
||||
}
|
||||
msg.MultiContent = append(msg.MultiContent, schema.ChatMessagePart{
|
||||
Type: mc.Type,
|
||||
ImageURL: &imageURLCopy,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
log.Debug().Interface("messages", msgs).Msg("log request messages")
|
||||
}
|
||||
|
||||
func logResponse(resp *schema.Message) {
|
||||
logger := log.Info().Str("role", string(resp.Role)).
|
||||
Str("content", resp.Content)
|
||||
if resp.ResponseMeta != nil {
|
||||
logger = logger.Interface("response_meta", resp.ResponseMeta)
|
||||
}
|
||||
if resp.Extra != nil {
|
||||
logger = logger.Interface("extra", resp.Extra)
|
||||
}
|
||||
logger.Msg("log response message")
|
||||
}
|
||||
|
||||
// appendConversationHistory adds a message to the conversation history
|
||||
func (p *Planner) appendConversationHistory(msg *schema.Message) {
|
||||
// for user image message:
|
||||
// - keep at most 4 user image messages
|
||||
// - delete the oldest user image message when the limit is reached
|
||||
if msg.Role == schema.User {
|
||||
// get all existing user messages
|
||||
userImgCount := 0
|
||||
firstUserImgIndex := -1
|
||||
|
||||
// calculate the number of user messages and find the index of the first user message
|
||||
for i, item := range p.history {
|
||||
if item.Role == schema.User {
|
||||
userImgCount++
|
||||
if firstUserImgIndex == -1 {
|
||||
firstUserImgIndex = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if there are already 4 user messages, delete the first one before adding the new message
|
||||
if userImgCount >= 4 && firstUserImgIndex >= 0 {
|
||||
// delete the first user message
|
||||
p.history = append(
|
||||
p.history[:firstUserImgIndex],
|
||||
p.history[firstUserImgIndex+1:]...,
|
||||
)
|
||||
}
|
||||
// add the new user message to the history
|
||||
p.history = append(p.history, msg)
|
||||
}
|
||||
|
||||
// for assistant message:
|
||||
// - keep at most the last 10 assistant messages
|
||||
if msg.Role == schema.Assistant {
|
||||
// add the new assistant message to the history
|
||||
p.history = append(p.history, msg)
|
||||
|
||||
// if there are more than 10 assistant messages, remove the oldest ones
|
||||
assistantMsgCount := 0
|
||||
for i := len(p.history) - 1; i >= 0; i-- {
|
||||
if p.history[i].Role == schema.Assistant {
|
||||
assistantMsgCount++
|
||||
if assistantMsgCount > 10 {
|
||||
p.history = append(p.history[:i], p.history[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse response
|
||||
parseActions, err := p.parser.Parse(msg.Content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse actions: %w", err)
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// processVLMResponse processes the VLM response and converts it to PlanningResult
|
||||
func processVLMResponse(actions []ParsedAction, size types.Size) (*PlanningResult, error) {
|
||||
log.Info().Msg("processing VLM response...")
|
||||
|
||||
if len(actions) == 0 {
|
||||
return nil, fmt.Errorf("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// validate and post-process each action
|
||||
for i := range actions {
|
||||
// validate action type
|
||||
switch actions[i].ActionType {
|
||||
case "click":
|
||||
if err := convertCoordinateAction(&actions[i], "startBox", size); err != nil {
|
||||
return nil, errors.Wrap(err, "convert coordinate action failed")
|
||||
}
|
||||
case "drag":
|
||||
if err := convertCoordinateAction(&actions[i], "startBox", size); err != nil {
|
||||
return nil, errors.Wrap(err, "convert coordinate action failed")
|
||||
}
|
||||
if err := convertCoordinateAction(&actions[i], "endBox", size); err != nil {
|
||||
return nil, errors.Wrap(err, "convert coordinate action failed")
|
||||
}
|
||||
case "type":
|
||||
validateTypeContent(&actions[i])
|
||||
case "wait", "finished", "call_user":
|
||||
// these actions do not need extra parameters
|
||||
default:
|
||||
log.Printf("warning: unknown action type: %s, will try to continue processing", actions[i].ActionType)
|
||||
}
|
||||
}
|
||||
|
||||
// extract action summary
|
||||
actionSummary := extractActionSummary(actions)
|
||||
|
||||
return &PlanningResult{
|
||||
NextActions: actions,
|
||||
ActionSummary: actionSummary,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractActionSummary extracts the summary from the actions
|
||||
func extractActionSummary(actions []ParsedAction) string {
|
||||
if len(actions) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// use the Thought of the first action as summary
|
||||
if actions[0].Thought != "" {
|
||||
return actions[0].Thought
|
||||
}
|
||||
|
||||
// if no Thought, generate summary from action type
|
||||
action := actions[0]
|
||||
switch action.ActionType {
|
||||
case "click":
|
||||
return "点击操作"
|
||||
case "drag":
|
||||
return "拖拽操作"
|
||||
case "type":
|
||||
content, _ := action.ActionInputs["content"].(string)
|
||||
if len(content) > 20 {
|
||||
content = content[:20] + "..."
|
||||
}
|
||||
return fmt.Sprintf("输入文本: %s", content)
|
||||
case "wait":
|
||||
return "等待操作"
|
||||
case "finished":
|
||||
return "完成操作"
|
||||
case "call_user":
|
||||
return "请求用户协助"
|
||||
default:
|
||||
return fmt.Sprintf("执行 %s 操作", action.ActionType)
|
||||
}
|
||||
}
|
||||
|
||||
func convertCoordinateAction(action *ParsedAction, boxField string, size types.Size) error {
|
||||
// The model generates a 2D coordinate output that represents relative positions.
|
||||
// To convert these values to image-relative coordinates, divide each component by 1000 to obtain values in the range [0,1].
|
||||
// The absolute coordinates required by the Action can be calculated by:
|
||||
// - X absolute = X relative × image width / 1000
|
||||
// - Y absolute = Y relative × image height / 1000
|
||||
|
||||
// get image width and height
|
||||
imageWidth := size.Width
|
||||
imageHeight := size.Height
|
||||
|
||||
box := action.ActionInputs[boxField]
|
||||
coords, ok := box.([]float64)
|
||||
if !ok {
|
||||
log.Error().Interface("inputs", action.ActionInputs).Msg("invalid action inputs")
|
||||
return fmt.Errorf("invalid action inputs")
|
||||
}
|
||||
|
||||
if len(coords) == 2 {
|
||||
coords[0] = math.Round((coords[0]/1000*float64(imageWidth))*10) / 10
|
||||
coords[1] = math.Round((coords[1]/1000*float64(imageHeight))*10) / 10
|
||||
} else if len(coords) == 4 {
|
||||
coords[0] = math.Round((coords[0]/1000*float64(imageWidth))*10) / 10
|
||||
coords[1] = math.Round((coords[1]/1000*float64(imageHeight))*10) / 10
|
||||
coords[2] = math.Round((coords[2]/1000*float64(imageWidth))*10) / 10
|
||||
coords[3] = math.Round((coords[3]/1000*float64(imageHeight))*10) / 10
|
||||
} else {
|
||||
log.Error().Interface("inputs", action.ActionInputs).Msg("invalid action inputs")
|
||||
return fmt.Errorf("invalid action inputs")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateTypeContent 验证输入文本内容
|
||||
func validateTypeContent(action *ParsedAction) {
|
||||
if content, ok := action.ActionInputs["content"]; !ok || content == "" {
|
||||
// default to empty string
|
||||
action.ActionInputs["content"] = ""
|
||||
log.Warn().Msg("type action missing content parameter, set to default")
|
||||
}
|
||||
}
|
||||
|
||||
// SavePositionImg saves an image with position markers
|
||||
func SavePositionImg(params struct {
|
||||
InputImgBase64 string
|
||||
Rect struct {
|
||||
X float64
|
||||
Y float64
|
||||
}
|
||||
OutputPath string
|
||||
}) error {
|
||||
// 解码Base64图像
|
||||
imgData := params.InputImgBase64
|
||||
// 如果包含了数据URL前缀，去掉它
|
||||
if strings.HasPrefix(imgData, "data:image/") {
|
||||
parts := strings.Split(imgData, ",")
|
||||
if len(parts) > 1 {
|
||||
imgData = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
// 解码Base64
|
||||
unbased, err := base64.StdEncoding.DecodeString(imgData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法解码Base64图像: %w", err)
|
||||
}
|
||||
|
||||
// 解码图像
|
||||
reader := bytes.NewReader(unbased)
|
||||
img, _, err := image.Decode(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法解码图像数据: %w", err)
|
||||
}
|
||||
|
||||
// 创建一个可以在其上绘制的图像
|
||||
bounds := img.Bounds()
|
||||
rgba := image.NewRGBA(bounds)
|
||||
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
|
||||
|
||||
// 在点击/拖动位置绘制标记
|
||||
markRadius := 10
|
||||
x, y := int(params.Rect.X), int(params.Rect.Y)
|
||||
|
||||
// 绘制红色圆圈
|
||||
for i := -markRadius; i <= markRadius; i++ {
|
||||
for j := -markRadius; j <= markRadius; j++ {
|
||||
if i*i+j*j <= markRadius*markRadius {
|
||||
if x+i >= 0 && x+i < bounds.Max.X && y+j >= 0 && y+j < bounds.Max.Y {
|
||||
rgba.Set(x+i, y+j, color.RGBA{255, 0, 0, 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 保存图像
|
||||
outFile, err := os.Create(params.OutputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法创建输出文件: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// 编码为PNG并保存
|
||||
if err := png.Encode(outFile, rgba); err != nil {
|
||||
return fmt.Errorf("无法编码和保存图像: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadImage loads image and returns base64 encoded string
|
||||
func loadImage(imagePath string) (base64Str string, size types.Size, err error) {
|
||||
// Read the image file
|
||||
imageFile, err := os.Open(imagePath)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
|
||||
}
|
||||
defer imageFile.Close()
|
||||
|
||||
// Decode the image to get its resolution
|
||||
imageData, format, err := image.Decode(imageFile)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
|
||||
}
|
||||
|
||||
// Get the resolution of the image
|
||||
width := imageData.Bounds().Dx()
|
||||
height := imageData.Bounds().Dy()
|
||||
size = types.Size{Width: width, Height: height}
|
||||
|
||||
// Convert image to base64
|
||||
buf := new(bytes.Buffer)
|
||||
if err := png.Encode(buf, imageData); err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
|
||||
}
|
||||
base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
|
||||
base64.StdEncoding.EncodeToString(buf.Bytes()))
|
||||
|
||||
return base64Str, size, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,363 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVLMPlanning(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
|
||||
1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
|
||||
2. 连接规则:
|
||||
- 两个相同的图案可以通过不超过三条直线连接。
|
||||
- 连接线可以水平或垂直，但不能穿过其他图案。
|
||||
- 连接线的转折次数不能超过两次。
|
||||
3. 游戏界面: 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
|
||||
4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
|
||||
5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
|
||||
6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。`
|
||||
|
||||
userInstruction += "\n\n请基于以上游戏规则，给出下一步可点击的两个图标坐标"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
UserInstruction: userInstruction,
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
MultiContent: []schema.ChatMessagePart{
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: imageBase64,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Size: size,
|
||||
}
|
||||
|
||||
// 执行规划
|
||||
result, err := planner.Call(opts)
|
||||
|
||||
// 验证结果
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.NotEmpty(t, result.NextActions)
|
||||
|
||||
// 验证动作
|
||||
action := result.NextActions[0]
|
||||
assert.NotEmpty(t, action.ActionType)
|
||||
assert.NotEmpty(t, action.Thought)
|
||||
|
||||
// 根据动作类型验证参数
|
||||
switch action.ActionType {
|
||||
case "click", "drag", "left_double", "right_single", "scroll":
|
||||
// 这些动作需要验证坐标
|
||||
assert.NotEmpty(t, action.ActionInputs["startBox"])
|
||||
|
||||
// 验证坐标格式
|
||||
coords, ok := action.ActionInputs["startBox"].([]float64)
|
||||
require.True(t, ok)
|
||||
require.True(t, len(coords) >= 2) // 至少有 x, y 坐标
|
||||
|
||||
// 验证坐标范围
|
||||
for _, coord := range coords {
|
||||
assert.GreaterOrEqual(t, coord, float64(0))
|
||||
}
|
||||
|
||||
case "type":
|
||||
// 验证文本内容
|
||||
assert.NotEmpty(t, action.ActionInputs["content"])
|
||||
|
||||
case "hotkey":
|
||||
// 验证按键
|
||||
assert.NotEmpty(t, action.ActionInputs["key"])
|
||||
|
||||
case "wait", "finished", "call_user":
|
||||
// 这些动作不需要额外参数
|
||||
|
||||
default:
|
||||
t.Fatalf("未知的动作类型: %s", action.ActionType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestXHSPlanning(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/xhs-feed.jpeg")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := "点击第二个帖子的作者头像"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
UserInstruction: userInstruction,
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
MultiContent: []schema.ChatMessagePart{
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: imageBase64,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Size: size,
|
||||
}
|
||||
|
||||
// 执行规划
|
||||
result, err := planner.Call(opts)
|
||||
|
||||
// 验证结果
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.NotEmpty(t, result.NextActions)
|
||||
|
||||
// 验证动作
|
||||
action := result.NextActions[0]
|
||||
assert.NotEmpty(t, action.ActionType)
|
||||
assert.NotEmpty(t, action.Thought)
|
||||
|
||||
// 根据动作类型验证参数
|
||||
switch action.ActionType {
|
||||
case "click", "drag", "left_double", "right_single", "scroll":
|
||||
// 这些动作需要验证坐标
|
||||
assert.NotEmpty(t, action.ActionInputs["startBox"])
|
||||
|
||||
// 验证坐标格式
|
||||
coords, ok := action.ActionInputs["startBox"].([]float64)
|
||||
require.True(t, ok)
|
||||
require.True(t, len(coords) >= 2) // 至少有 x, y 坐标
|
||||
|
||||
// 验证坐标范围
|
||||
for _, coord := range coords {
|
||||
assert.GreaterOrEqual(t, coord, float64(0))
|
||||
}
|
||||
|
||||
case "type":
|
||||
// 验证文本内容
|
||||
assert.NotEmpty(t, action.ActionInputs["content"])
|
||||
|
||||
case "hotkey":
|
||||
// 验证按键
|
||||
assert.NotEmpty(t, action.ActionInputs["key"])
|
||||
|
||||
case "wait", "finished", "call_user":
|
||||
// 这些动作不需要额外参数
|
||||
|
||||
default:
|
||||
t.Fatalf("未知的动作类型: %s", action.ActionType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatList(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/chat_list.jpeg")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := "请结合图片的文字信息，请告诉我一共有多少个群聊，哪些群聊右下角有绿点"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
UserInstruction: userInstruction,
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
MultiContent: []schema.ChatMessagePart{
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: imageBase64,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Size: size,
|
||||
}
|
||||
|
||||
// 执行规划
|
||||
result, err := planner.Call(opts)
|
||||
|
||||
// 验证结果
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
}
|
||||
|
||||
func TestValidateInput(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/popup_risk_warning.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
opts *PlanningOptions
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "valid input",
|
||||
opts: &PlanningOptions{
|
||||
UserInstruction: "点击继续使用按钮",
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
MultiContent: []schema.ChatMessagePart{
|
||||
{
|
||||
Type: "image_url",
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: imageBase64,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Size: size,
|
||||
},
|
||||
wantErr: nil,
|
||||
},
|
||||
{
|
||||
name: "empty instruction",
|
||||
opts: &PlanningOptions{
|
||||
UserInstruction: "",
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
Content: "",
|
||||
},
|
||||
Size: size,
|
||||
},
|
||||
wantErr: ErrEmptyInstruction,
|
||||
},
|
||||
{
|
||||
name: "empty conversation history",
|
||||
opts: &PlanningOptions{
|
||||
UserInstruction: "点击立即卸载按钮",
|
||||
Message: &schema.Message{},
|
||||
},
|
||||
wantErr: ErrNoConversationHistory,
|
||||
},
|
||||
{
|
||||
name: "invalid image data",
|
||||
opts: &PlanningOptions{
|
||||
UserInstruction: "点击继续使用按钮",
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
Content: "no image",
|
||||
},
|
||||
Size: size,
|
||||
},
|
||||
wantErr: ErrInvalidImageData,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateInput(tt.opts)
|
||||
if tt.wantErr != nil {
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, tt.wantErr, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessVLMResponse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
actions []ParsedAction
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid response",
|
||||
actions: []ParsedAction{
|
||||
{
|
||||
ActionType: "click",
|
||||
ActionInputs: map[string]interface{}{
|
||||
"startBox": []float64{0.5, 0.5},
|
||||
},
|
||||
Thought: "点击中心位置",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty actions",
|
||||
actions: []ParsedAction{},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
size := types.Size{
|
||||
Width: 1000,
|
||||
Height: 1000,
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := processVLMResponse(tt.actions, size)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, result)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, tt.actions, result.NextActions)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSavePositionImg(t *testing.T) {
|
||||
imageBase64, _, err := loadImage("testdata/popup_risk_warning.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
params := struct {
|
||||
InputImgBase64 string
|
||||
Rect struct {
|
||||
X float64
|
||||
Y float64
|
||||
}
|
||||
OutputPath string
|
||||
}{
|
||||
InputImgBase64: imageBase64,
|
||||
Rect: struct {
|
||||
X float64
|
||||
Y float64
|
||||
}{
|
||||
X: 100,
|
||||
Y: 100,
|
||||
},
|
||||
OutputPath: "testdata/output.png",
|
||||
}
|
||||
|
||||
err = SavePositionImg(params)
|
||||
require.NoError(t, err)
|
||||
|
||||
// cleanup
|
||||
defer os.Remove(params.OutputPath)
|
||||
}
|
||||
|
||||
func TestLoadImage(t *testing.T) {
|
||||
// Test PNG image
|
||||
pngBase64, pngSize, err := loadImage("testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, pngBase64)
|
||||
assert.Greater(t, pngSize.Width, 0)
|
||||
assert.Greater(t, pngSize.Height, 0)
|
||||
|
||||
// Test JPEG image
|
||||
jpegBase64, jpegSize, err := loadImage("testdata/xhs-feed.jpeg")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, jpegBase64)
|
||||
assert.Greater(t, jpegSize.Width, 0)
|
||||
assert.Greater(t, jpegSize.Height, 0)
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package ai
|
||||
|
||||
const uiTarsPlanningPrompt = `
|
||||
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
|
||||
|
||||
## Output Format
|
||||
Thought: ...
|
||||
Action: ...
|
||||
|
||||
## Action Space
|
||||
click(start_box='[x1,y1]')
|
||||
long_press(start_box='[x1,y1]', time='')
|
||||
type(content='')
|
||||
drag(start_box='[x1,y1]', end_box='[x2,y2]')
|
||||
press_home()
|
||||
press_back()
|
||||
finished(content='') # Submit the task regardless of whether it succeeds or fails.
|
||||
|
||||
## Note
|
||||
- Use Chinese in Thought part.
|
||||
- Write a small plan and finally summarize your next action (with its target element) in one sentence in Thought part.
|
||||
|
||||
## User Instruction
|
||||
`
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 393 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 390 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 295 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 437 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 407 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 123 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 649 KiB |
|
|
@ -944,7 +944,88 @@ func (ad *ADBDriver) OpenUrl(url string) (err error) {
|
|||
|
||||
func (ad *ADBDriver) PushImage(localPath string) error {
|
||||
log.Info().Str("localPath", localPath).Msg("ADBDriver.PushImage")
|
||||
remotePath := path.Join("/sdcard/DCIM/Camera/", path.Base(localPath))
|
||||
remoteDir := "/sdcard/DCIM/Camera/"
|
||||
return ad.PushFile(localPath, remoteDir)
|
||||
}
|
||||
|
||||
// PullImages pulls all images from device's DCIM/Camera directory to local directory
|
||||
func (ad *ADBDriver) PullImages(localDir string) error {
|
||||
log.Info().Str("localDir", localDir).Msg("ADBDriver.PullImages")
|
||||
remoteDir := "/sdcard/DCIM/Camera/"
|
||||
|
||||
// create local directory if not exists
|
||||
if err := os.MkdirAll(localDir, 0o755); err != nil {
|
||||
return fmt.Errorf("failed to create local directory: %w", err)
|
||||
}
|
||||
|
||||
files, err := ad.Device.List(remoteDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list directory %s: %w", remoteDir, err)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
// filter image files by extension
|
||||
ext := strings.ToLower(path.Ext(file.Name))
|
||||
if !isImageFile(ext) {
|
||||
continue
|
||||
}
|
||||
|
||||
remotePath := path.Join(remoteDir, file.Name)
|
||||
localPath := path.Join(localDir, file.Name)
|
||||
|
||||
// check if file already exists
|
||||
if _, err := os.Stat(localPath); err == nil {
|
||||
log.Debug().Str("localPath", localPath).Msg("file already exists, skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// create local file
|
||||
f, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("localPath", localPath).Msg("failed to create local file")
|
||||
continue
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// pull image file
|
||||
if err := ad.Device.Pull(remotePath, f); err != nil {
|
||||
log.Error().Err(err).
|
||||
Str("remotePath", remotePath).
|
||||
Str("localPath", localPath).
|
||||
Msg("failed to pull image")
|
||||
continue // continue with next file
|
||||
}
|
||||
log.Info().
|
||||
Str("remotePath", remotePath).
|
||||
Str("localPath", localPath).
|
||||
Msg("image pulled successfully")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isImageFile checks if the file extension is an image format
|
||||
func isImageFile(ext string) bool {
|
||||
imageExts := map[string]bool{
|
||||
".jpg": true,
|
||||
".jpeg": true,
|
||||
".png": true,
|
||||
".gif": true,
|
||||
".bmp": true,
|
||||
".webp": true,
|
||||
".heic": true,
|
||||
}
|
||||
return imageExts[ext]
|
||||
}
|
||||
|
||||
func (ad *ADBDriver) ClearImages() error {
|
||||
log.Info().Msg("ADBDriver.ClearImages")
|
||||
_, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ad *ADBDriver) PushFile(localPath string, remoteDir string) error {
|
||||
log.Info().Str("localPath", localPath).Str("remoteDir", remoteDir).Msg("ADBDriver.PushFile")
|
||||
remotePath := path.Join(remoteDir, path.Base(localPath))
|
||||
if err := ad.Device.PushFile(localPath, remotePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -955,9 +1036,60 @@ func (ad *ADBDriver) PushImage(localPath string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (ad *ADBDriver) ClearImages() error {
|
||||
log.Info().Msg("ADBDriver.ClearImages")
|
||||
_, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*")
|
||||
func (ad *ADBDriver) PullFiles(localDir string, remoteDirs ...string) error {
|
||||
log.Info().Str("localDir", localDir).Strs("remoteDirs", remoteDirs).Msg("ADBDriver.PullFiles")
|
||||
|
||||
// create local directory if not exists
|
||||
if err := os.MkdirAll(localDir, 0o755); err != nil {
|
||||
return fmt.Errorf("failed to create local directory: %w", err)
|
||||
}
|
||||
|
||||
for _, remoteDir := range remoteDirs {
|
||||
files, err := ad.Device.List(remoteDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list directory %s: %w", remoteDir, err)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
remotePath := path.Join(remoteDir, file.Name)
|
||||
localPath := path.Join(localDir, file.Name)
|
||||
|
||||
// check if file already exists
|
||||
if _, err := os.Stat(localPath); err == nil {
|
||||
log.Debug().Str("localPath", localPath).Msg("file already exists, skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
// create local file
|
||||
f, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("localPath", localPath).Msg("failed to create local file")
|
||||
continue
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// pull image file
|
||||
if err := ad.Device.Pull(remotePath, f); err != nil {
|
||||
log.Error().Err(err).
|
||||
Str("remotePath", remotePath).
|
||||
Str("localPath", localPath).
|
||||
Msg("failed to pull file")
|
||||
continue // continue with next file
|
||||
}
|
||||
log.Info().
|
||||
Str("remotePath", remotePath).
|
||||
Str("localPath", localPath).
|
||||
Msg("file pulled successfully")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ad *ADBDriver) ClearFiles(paths ...string) error {
|
||||
log.Info().Strs("paths", paths).Msg("ADBDriver.ClearFiles")
|
||||
for _, path := range paths {
|
||||
_, _ = ad.Device.RunShellCommand("rm", "-rf", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ func setupADBDriverExt(t *testing.T) *XTDriver {
|
|||
driver, err := device.NewDriver()
|
||||
require.Nil(t, err)
|
||||
return NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM),
|
||||
ai.WithLLMService(ai.LLMServiceTypeUITARS),
|
||||
)
|
||||
}
|
||||
|
||||
func setupUIA2DriverExt(t *testing.T) *XTDriver {
|
||||
|
|
@ -258,6 +260,9 @@ func TestDriver_ADB_PushImage(t *testing.T) {
|
|||
err = driver.PushImage(path)
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = driver.PullImages("./test")
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = driver.ClearImages()
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -444,11 +444,27 @@ func (wd *BrowserDriver) AppClear(packageName string) error {
|
|||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) PushImage(localPath string) error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) PullImages(localDir string) error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) ClearImages() error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) PushImage(localPath string) error {
|
||||
func (wd *BrowserDriver) PushFile(localPath string, remoteDir string) error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) PullFiles(localDir string, remoteDirs ...string) error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) ClearFiles(paths ...string) error {
|
||||
return errors.New("not support")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -68,8 +68,14 @@ type IDriver interface {
|
|||
|
||||
// image related
|
||||
PushImage(localPath string) error
|
||||
PullImages(localDir string) error
|
||||
ClearImages() error
|
||||
|
||||
// files related
|
||||
PushFile(localPath string, remoteDir string) error
|
||||
PullFiles(localDir string, remoteDirs ...string) error
|
||||
ClearFiles(paths ...string) error
|
||||
|
||||
// triggers the log capture and returns the log entries
|
||||
StartCaptureLog(identifier ...string) error
|
||||
StopCaptureLog() (result interface{}, err error)
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ const (
|
|||
ACTION_Input ActionMethod = "input"
|
||||
ACTION_Back ActionMethod = "back"
|
||||
ACTION_KeyCode ActionMethod = "keycode"
|
||||
ACTION_AIAction ActionMethod = "ai_action" // action with ai
|
||||
|
||||
// custom actions
|
||||
ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap
|
||||
|
|
@ -64,12 +65,14 @@ const (
|
|||
SelectorLabel string = "ui_label"
|
||||
SelectorOCR string = "ui_ocr"
|
||||
SelectorImage string = "ui_image"
|
||||
SelectorAI string = "ui_ai" // ui query with ai
|
||||
SelectorForegroundApp string = "ui_foreground_app"
|
||||
// assertions
|
||||
AssertionEqual string = "equal"
|
||||
AssertionNotEqual string = "not_equal"
|
||||
AssertionExists string = "exists"
|
||||
AssertionNotExists string = "not_exists"
|
||||
AssertionAI string = "ai_assert" // assert with ai
|
||||
)
|
||||
|
||||
type MobileAction struct {
|
||||
|
|
@ -281,6 +284,11 @@ func (dExt *XTDriver) DoAction(action MobileAction) (err error) {
|
|||
fn := action.Fn
|
||||
fn()
|
||||
return nil
|
||||
case ACTION_AIAction:
|
||||
if prompt, ok := action.Params.(string); ok {
|
||||
return dExt.AIAction(prompt, action.GetOptions()...)
|
||||
}
|
||||
return fmt.Errorf("invalid %s params: %v", ACTION_AIAction, action.Params)
|
||||
default:
|
||||
log.Warn().Str("action", string(action.Method)).Msg("action not implemented")
|
||||
return errors.Wrapf(code.InvalidCaseError,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
package uixt
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func (dExt *XTDriver) StartToGoal(text string, opts ...option.ActionOption) error {
|
||||
options := option.NewActionOptions(opts...)
|
||||
var attempt int
|
||||
for {
|
||||
attempt++
|
||||
log.Info().Int("attempt", attempt).Msg("planning attempt")
|
||||
if err := dExt.AIAction(text, opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if options.MaxRetryTimes > 1 && attempt >= options.MaxRetryTimes {
|
||||
return errors.New("reached max retry times")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dExt *XTDriver) AIAction(text string, opts ...option.ActionOption) error {
|
||||
// plan next action
|
||||
result, err := dExt.PlanNextAction(text, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do actions
|
||||
for _, action := range result.NextActions {
|
||||
switch action.ActionType {
|
||||
case ai.ActionTypeClick:
|
||||
point := action.ActionInputs["startBox"].([]float64)
|
||||
if err := dExt.TapAbsXY(point[0], point[1], opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
case ai.ActionTypeFinished:
|
||||
log.Info().Msg("ai action done")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (dExt *XTDriver) AIAssert(text string, opts ...option.ActionOption) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dExt *XTDriver) PlanNextAction(text string, opts ...option.ActionOption) (*ai.PlanningResult, error) {
|
||||
if dExt.LLMService == nil {
|
||||
return nil, errors.New("LLM service is not initialized")
|
||||
}
|
||||
|
||||
compressedBufSource, err := dExt.GetScreenShotBuffer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// convert buffer to base64 string
|
||||
screenShotBase64 := "data:image/jpeg;base64," +
|
||||
base64.StdEncoding.EncodeToString(compressedBufSource.Bytes())
|
||||
|
||||
// save screenshot to file
|
||||
imagePath := filepath.Join(
|
||||
config.GetConfig().ScreenShotsPath,
|
||||
fmt.Sprintf("%s.jpeg", builtin.GenNameWithTimestamp("%d_screenshot")),
|
||||
)
|
||||
go func() {
|
||||
err := saveScreenShot(compressedBufSource, imagePath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("save screenshot file failed")
|
||||
}
|
||||
}()
|
||||
|
||||
size, err := dExt.IDriver.WindowSize()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.DeviceGetInfoError, err.Error())
|
||||
}
|
||||
|
||||
planningOpts := &ai.PlanningOptions{
|
||||
UserInstruction: text,
|
||||
Message: &schema.Message{
|
||||
Role: schema.User,
|
||||
MultiContent: []schema.ChatMessagePart{
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: screenShotBase64,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Size: size,
|
||||
}
|
||||
|
||||
result, err := dExt.LLMService.Call(planningOpts)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get next action from planner")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
_ "image/png"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -45,10 +44,7 @@ func (s *ScreenResult) FilterTextsByScope(x1, y1, x2, y2 float64) ai.OCRTexts {
|
|||
})
|
||||
}
|
||||
|
||||
// GetScreenResult takes a screenshot, returns the image recognition result
|
||||
func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult *ScreenResult, err error) {
|
||||
screenshotOptions := option.NewActionOptions(opts...)
|
||||
|
||||
func (dExt *XTDriver) GetScreenShotBuffer() (compressedBufSource *bytes.Buffer, err error) {
|
||||
// take screenshot
|
||||
bufSource, err := dExt.ScreenShot()
|
||||
if err != nil {
|
||||
|
|
@ -63,6 +59,19 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
|
|||
"compress screenshot failed %v", err)
|
||||
}
|
||||
|
||||
return compressBufSource, nil
|
||||
}
|
||||
|
||||
// GetScreenResult takes a screenshot, returns the image recognition result
|
||||
func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult *ScreenResult, err error) {
|
||||
// get compressed screenshot buffer
|
||||
compressBufSource, err := dExt.GetScreenShotBuffer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
screenshotOptions := option.NewActionOptions(opts...)
|
||||
|
||||
// save compressed screenshot to file
|
||||
var fileName string
|
||||
optionsList := screenshotOptions.List()
|
||||
|
|
@ -81,8 +90,6 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
|
|||
err := saveScreenShot(compressBufSource, imagePath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("save screenshot file failed")
|
||||
} else {
|
||||
log.Info().Str("path", imagePath).Msg("screenshot saved")
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
|
|||
|
|
@ -123,6 +123,47 @@ func TestDriverExt_TapByOCR(t *testing.T) {
|
|||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestDriverExt_TapByLLM(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
err := driver.AIAction("点击第一个帖子的作者头像")
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestDriverExt_StartToGoal(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
|
||||
1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
|
||||
2. 连接规则:
|
||||
- 两个相同的图案可以通过不超过三条直线连接。
|
||||
- 连接线可以水平或垂直，但不能斜线，也不能跨过其他图案。
|
||||
- 连接线的转折次数不能超过两次。
|
||||
3. 游戏界面:
|
||||
- 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
|
||||
- 图案或图标在未选中状态下背景为白色，选中状态下背景为绿色。
|
||||
4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
|
||||
5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
|
||||
6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。
|
||||
|
||||
注意事项：
|
||||
1、当连接错误时，顶部的红心会减少一个，需及时调整策略，避免红心变为0个后游戏失败
|
||||
2、不要连续 2 次点击同一个图案
|
||||
3、不要犯重复的错误
|
||||
`
|
||||
|
||||
userInstruction += "\n\n请严格按照以上游戏规则，开始游戏；注意，请只做点击操作"
|
||||
|
||||
err := driver.StartToGoal(userInstruction)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestDriverExt_PlanNextAction(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
result, err := driver.PlanNextAction("启动抖音")
|
||||
assert.Nil(t, err)
|
||||
t.Log(result)
|
||||
}
|
||||
|
||||
func TestDriverExt_prepareSwipeAction(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
|
|
|
|||
|
|
@ -185,8 +185,12 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st
|
|||
switch check {
|
||||
case SelectorOCR:
|
||||
err = dExt.assertOCR(expected, assert)
|
||||
// case SelectorAI:
|
||||
// // TODO
|
||||
case SelectorForegroundApp:
|
||||
err = dExt.assertForegroundApp(expected, assert)
|
||||
default:
|
||||
return fmt.Errorf("validator %s not implemented", check)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -198,7 +202,8 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st
|
|||
return err
|
||||
}
|
||||
|
||||
log.Info().Str("assert", assert).Str("expect", expected).Msg("validate success")
|
||||
log.Info().Str("check", check).Str("assert", assert).
|
||||
Str("expect", expected).Msg("validate success")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -288,7 +288,27 @@ func (hd *HDCDriver) PushImage(localPath string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (hd *HDCDriver) PullImages(localDir string) error {
|
||||
log.Warn().Msg("PullImages not implemented in HDCDriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hd *HDCDriver) ClearImages() error {
|
||||
log.Warn().Msg("ClearImages not implemented in HDCDriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hd *HDCDriver) PushFile(localPath string, remoteDir string) error {
|
||||
log.Warn().Msg("PushFile not implemented in HDCDriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hd *HDCDriver) PullFiles(localDir string, remoteDirs ...string) error {
|
||||
log.Warn().Msg("PullFiles not implemented in HDCDriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hd *HDCDriver) ClearFiles(paths ...string) error {
|
||||
log.Warn().Msg("ClearFiles not implemented in HDCDriver")
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -989,6 +989,11 @@ func (wd *WDADriver) PushImage(localPath string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (wd *WDADriver) PullImages(localDir string) error {
|
||||
log.Warn().Msg("PullImages not implemented in WDADriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wd *WDADriver) ClearImages() error {
|
||||
log.Info().Msg("WDADriver.ClearImages")
|
||||
data := map[string]interface{}{}
|
||||
|
|
@ -997,6 +1002,21 @@ func (wd *WDADriver) ClearImages() error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (wd *WDADriver) PushFile(localPath string, remoteDir string) error {
|
||||
log.Warn().Msg("PushFile not implemented in WDADriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wd *WDADriver) PullFiles(localDir string, remoteDirs ...string) error {
|
||||
log.Warn().Msg("PullFiles not implemented in WDADriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wd *WDADriver) ClearFiles(paths ...string) error {
|
||||
log.Warn().Msg("ClearFiles not implemented in WDADriver")
|
||||
return nil
|
||||
}
|
||||
|
||||
type wdaResponse struct {
|
||||
Status int `json:"status"`
|
||||
SessionID string `json:"sessionId"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue