Merge branch 'ai-ui-tars' into 'v5'

新增 LLM 能力，集成 UI-Tars

See merge request iesqa/httprunner!77

.gitignore

@@ -7,6 +7,7 @@
 
 # Test binary, built with `go test -c`
 *.test
+.env
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

Makefile

@@ -16,10 +16,12 @@ bump: ## bump hrp version, e.g. make bump version=4.0.0
 .PHONY: build
 build: ## build hrp cli tool
 	@echo "[info] build hrp cli tool"
+	go mod tidy
 	go build -ldflags "\
 		-s -w \
 		-X 'github.com/httprunner/httprunner/v5/internal/version.GitCommit=$(shell git rev-parse HEAD)' \
 		-X 'github.com/httprunner/httprunner/v5/internal/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)' \
+		-X 'github.com/httprunner/httprunner/v5/internal/version.GitAuthor=$(shell git log -1 --pretty=format:%an)' \
 		-X 'github.com/httprunner/httprunner/v5/internal/version.BuildTime=$(shell date "+%y%m%d%H%M")'" \
 		-o output/hrp ./cmd/cli
 	./output/hrp -v

code/code.go

@@ -104,7 +104,8 @@ var (
 	CVResponseError          = errors.New("CV parse response error")  // 83
 	CVResultNotFoundError    = errors.New("CV result not found")      // 84
 
-	StateUnknowError = errors.New("detect state failed") // 85
+	LLMEnvMissedError = errors.New("LLM env missed error") // 85
+	StateUnknowError  = errors.New("detect state failed")  // 89
 )
 
 // trackings related: [90, 100)
@@ -199,7 +200,8 @@ var errorsMap = map[error]int{
 	CVServiceConnectionError: 82,
 	CVResponseError:          83,
 	CVResultNotFoundError:    84,
-	StateUnknowError:         85,
+	LLMEnvMissedError:        85,
+	StateUnknowError:         89,
 
 	// trackings related
 	TrackingGetError:   90,

go.mod

@@ -5,6 +5,8 @@ go 1.22.0
 require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/andybalholm/brotli v1.0.4
+	github.com/cloudwego/eino v0.3.16
+	github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c
 	github.com/danielpaulus/go-ios v1.0.161
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/fatih/color v1.16.0
@@ -16,29 +18,33 @@ require (
 	github.com/httprunner/funplugin v0.5.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/jmespath/go-jmespath v0.4.0
+	github.com/joho/godotenv v1.5.1
 	github.com/json-iterator/go v1.1.12
 	github.com/maja42/goval v1.2.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.30.0
+	github.com/rs/zerolog v1.33.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	golang.org/x/net v0.26.0
-	golang.org/x/text v0.16.0
+	golang.org/x/text v0.21.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic v1.12.2 // indirect
-	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/getkin/kin-openapi v0.118.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -53,11 +59,13 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/goph/emperror v0.17.2 // indirect
 	github.com/grandcat/zeroconf v1.0.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-plugin v1.4.10 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/invopop/yaml v0.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -69,29 +77,35 @@ require (
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/nikolalohinski/gonja v1.5.3 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/perimeterx/marshmallow v1.1.4 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
 	github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sashabaranov/go-openai v1.32.5 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/yargevad/filepathx v1.0.0 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
-	go.uber.org/mock v0.3.0 // indirect
+	go.uber.org/mock v0.4.0 // indirect
-	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/arch v0.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect

go.sum

@@ -1,18 +1,33 @@
 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
-github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
-github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
-github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
+github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/bytedance/mockey v1.2.13 h1:jokWZAm/pUEbD939Rhznz615MKUCZNuvCFQlJ2+ntoo=
+github.com/bytedance/mockey v1.2.13/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY=
+github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg=
+github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM=
+github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
 github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
+github.com/cloudwego/eino v0.3.16 h1:ASN8zISyoEdjEsPnIw5GazSHtbNY97NDthQ2B69yiZw=
+github.com/cloudwego/eino v0.3.16/go.mod h1:+kmJimGEcKuSI6OKhet7kBedkm1WUZS3H1QRazxgWUo=
+github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c h1:04WQpGikdQv6fh5wzMYSQhO0SJraV8+xcb9VQ00+HX4=
+github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c/go.mod h1:YGP4q3uspj5qhkv3CnvlEPSo0YGeWpvkkTUHHpLExas=
+github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 h1:fgvkmTqAalDfjdy3b6Ur2mh/KEwB9L2uvqS4MFgTOqc=
+github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955/go.mod h1:6CThw1XQx/ASXNt31yuvp0X4Yp4GprknQuIvP9VKDpw=
 github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -26,6 +41,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0=
 github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -33,14 +50,20 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM=
+github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo=
 github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
 github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
@@ -66,9 +89,13 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
+github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -84,6 +111,11 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18=
+github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic=
+github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
+github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
@@ -94,12 +126,15 @@ github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQ
 github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
 github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/httprunner/funplugin v0.5.5 h1:VU1a6kj1AsJ/ucIhhI5NLHXOP4xnW2JGgk50vBV3Zis=
 github.com/httprunner/funplugin v0.5.5/go.mod h1:YZzBBSOSdLZEpHZz0P2E5SOQ+o1+Fbn30oWS4RGHBz0=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
+github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
@@ -109,14 +144,20 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -140,8 +181,11 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
 github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
@@ -154,20 +198,30 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c=
+github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
 github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw=
+github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
 github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
 github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -179,15 +233,25 @@ github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55/go.mod h1:PeN7k
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
-github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sashabaranov/go-openai v1.32.5 h1:/eNVa8KzlE7mJdKPZDj6886MUzZQjoVHyn0sLvIt5qA=
+github.com/sashabaranov/go-openai v1.32.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI=
+github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg=
+github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
+github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
+github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
+github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
@@ -195,10 +259,12 @@ github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJ
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -207,31 +273,39 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 h1:aeN+ghOV0b2VCmKKO3gqnDQ8mLbpABZgRR2FVYx4ouI=
 github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9/go.mod h1:roo6cZ/uqpwKMuvPG0YmzI5+AmUiMWfjCBZpGXqbTxE=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
+github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
+github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
-go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
-go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4=
-golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
+golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -239,9 +313,12 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
 golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -258,13 +335,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -288,13 +368,17 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5 h1:DOUDfNS+CFMM46k18FRF5k/0yz5NhZYMiUQxf4xglIU=
@@ -302,6 +386,5 @@ gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5/go.mod h1:NQHVAzMwvZ+Qe3ElS
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE=
 software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ=

internal/version/VERSION

@@ -1 +1 @@
-v5.0.0-beta-2503251404
+v5.0.0-beta-2503311103

internal/version/version.go

@@ -19,6 +19,7 @@ var (
 	GitCommit = "unknown"
 	GitBranch = "unknown"
 	BuildTime = "unknown"
+	GitAuthor = "unknown"
 )
 
 func GetVersionInfo() string {
@@ -26,6 +27,6 @@ func GetVersionInfo() string {
 	if len(commitID) > 8 {
 		commitID = commitID[:8]
 	}
-	return fmt.Sprintf("%s (branch=%s, commit=%s, build=%s)",
+	return fmt.Sprintf("%s (branch=%s, commit=%s, author=%s, build=%s)",
-		VERSION, GitBranch, commitID, BuildTime)
+		VERSION, GitBranch, commitID, GitAuthor, BuildTime)
 }

step_ui.go

@@ -164,6 +164,18 @@ func (s *StepMobile) TapByUITypes(opts ...option.ActionOption) *StepMobile {
 	return s
 }
 
+// AIAction do actions with VLM
+func (s *StepMobile) AIAction(prompt string, opts ...option.ActionOption) *StepMobile {
+	action := uixt.MobileAction{
+		Method:  uixt.ACTION_AIAction,
+		Params:  prompt,
+		Options: option.NewActionOptions(opts...),
+	}
+
+	s.obj().Actions = append(s.obj().Actions, action)
+	return s
+}
+
 // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates
 func (s *StepMobile) DoubleTapXY(x, y float64, opts ...option.ActionOption) *StepMobile {
 	s.obj().Actions = append(s.obj().Actions, uixt.MobileAction{
@@ -527,6 +539,21 @@ func (s *StepMobileUIValidation) AssertImageNotExists(expectedImagePath string,
 	return s
 }
 
+func (s *StepMobileUIValidation) AssertAI(prompt string, msg ...string) *StepMobileUIValidation {
+	v := Validator{
+		Check:  uixt.SelectorAI,
+		Assert: uixt.AssertionAI,
+		Expect: prompt,
+	}
+	if len(msg) > 0 {
+		v.Message = msg[0]
+	} else {
+		v.Message = fmt.Sprintf("assert ai prompt [%s] failed", prompt)
+	}
+	s.Validators = append(s.Validators, v)
+	return s
+}
+
 func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg ...string) *StepMobileUIValidation {
 	v := Validator{
 		Check:  uixt.SelectorForegroundApp,

tests/step_ui_test.go

@@ -80,3 +80,21 @@ func TestAndroidAction(t *testing.T) {
 	err := hrp.NewRunner(t).Run(testCase)
 	assert.Nil(t, err)
 }
+
+func TestAIAction(t *testing.T) {
+	testCase := &hrp.TestCase{
+		Config: hrp.NewConfig("run ui action with ai"),
+		TestSteps: []hrp.IStep{
+			hrp.NewStep("launch settings").
+				Android().AIAction("进入手机系统设置").
+				Validate().
+				AssertAI("当前位于手机设置页面"),
+			hrp.NewStep("turn on fly mode").
+				Android().AIAction("开启飞行模式").
+				Validate().
+				AssertAI("飞行模式已开启"),
+		},
+	}
+	err := hrp.NewRunner(t).Run(testCase)
+	assert.Nil(t, err)
+}

uixt/ai/ai.go

@@ -1,6 +1,7 @@
 package ai
 
 import (
+	"context"
 	"os"
 
 	"github.com/rs/zerolog/log"
@@ -46,11 +47,17 @@ func WithCVService(service CVServiceType) AIServiceOption {
 type LLMServiceType string
 
 const (
+	LLMServiceTypeUITARS     LLMServiceType = "ui-tars"
 	LLMServiceTypeGPT4o      LLMServiceType = "gpt-4o"
 	LLMServiceTypeDeepSeekV3 LLMServiceType = "deepseek-v3"
 )
 
 func WithLLMService(service LLMServiceType) AIServiceOption {
+	if err := checkEnvLLM(); err != nil {
+		log.Error().Err(err).Msg("check LLM env failed")
+		os.Exit(code.GetErrorCode(err))
+	}
+
 	return func(opts *AIServices) {
 		if service == LLMServiceTypeGPT4o {
 			var err error
@@ -60,5 +67,12 @@ func WithLLMService(service LLMServiceType) AIServiceOption {
 				os.Exit(code.GetErrorCode(err))
 			}
 		}
+		if service == LLMServiceTypeUITARS {
+			var err error
+			opts.ILLMService, err = NewPlanner(context.Background())
+			if err != nil {
+				log.Error().Err(err).Msg("init ui-tars llm service failed")
+			}
+		}
 	}
 }

uixt/ai/ai_test.go

@@ -1,11 +1,13 @@
 package ai
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestOption(t *testing.T) {
 	options := NewAIService(
 		WithCVService(CVServiceTypeOpenCV),
-		WithLLMService(LLMServiceTypeDeepSeekV3),
+		WithLLMService(LLMServiceTypeUITARS),
 	)
 	t.Log(options)
 }

uixt/ai/cv_vedem.go

@@ -30,7 +30,7 @@ type APIResponseImage struct {
 }
 
 func NewVEDEMImageService() (*vedemCVService, error) {
-	if err := checkEnv(); err != nil {
+	if err := checkEnvCV(); err != nil {
 		return nil, err
 	}
 	return &vedemCVService{}, nil
@@ -230,7 +230,7 @@ func (s *vedemCVService) ReadFromBuffer(imageBuf *bytes.Buffer, opts ...option.A
 	return imageResult, nil
 }
 
-func checkEnv() error {
+func checkEnvCV() error {
 	vedemImageURL := os.Getenv("VEDEM_IMAGE_URL")
 	if vedemImageURL == "" {
 		return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_URL missed")

uixt/ai/env.go

@@ -0,0 +1,240 @@
+package ai
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/cloudwego/eino-ext/components/model/openai"
+	"github.com/httprunner/httprunner/v5/code"
+	"github.com/joho/godotenv"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	defaultTimeout = 60 * time.Second
+)
+
+type OpenAIInitConfig struct {
+	ReportURL string            `json:"REPORT_SERVER_URL"`
+	Headers   map[string]string `json:"defaultHeaders"`
+}
+
+const (
+	EnvOpenAIBaseURL        = "OPENAI_BASE_URL"
+	EnvOpenAIAPIKey         = "OPENAI_API_KEY"
+	EnvModelName            = "LLM_MODEL_NAME"
+	EnvOpenAIInitConfigJSON = "OPENAI_INIT_CONFIG_JSON"
+)
+
+var once sync.Once
+
+// loadEnv loads environment variables from .env file
+// it will search for .env file from current working directory upward recursively
+func loadEnv() {
+	once.Do(func() {
+		// get current working directory
+		cwd, err := os.Getwd()
+		if err != nil {
+			panic(err)
+		}
+
+		// locate .env file from current working directory upward recursively
+		envPath := cwd
+		for {
+			envFile := filepath.Join(envPath, ".env")
+			if _, err := os.Stat(envFile); err == nil {
+				// found .env file
+				// override existing env variables
+				err = godotenv.Overload(envFile)
+				if err != nil {
+					log.Fatal().Err(err).
+						Str("path", envFile).Msg("overload env file failed")
+				}
+				log.Info().Str("path", envFile).Msg("overload env success")
+				return
+			}
+
+			// reached root directory
+			parent := filepath.Dir(envPath)
+			if parent == envPath {
+				log.Info().Msg("no .env file found from current directory to root")
+				return
+			}
+			envPath = parent
+		}
+	})
+}
+
+func checkEnvLLM() error {
+	loadEnv()
+	openaiBaseURL := os.Getenv("OPENAI_BASE_URL")
+	if openaiBaseURL == "" {
+		return errors.Wrap(code.LLMEnvMissedError, "OPENAI_BASE_URL missed")
+	}
+	log.Info().Str("OPENAI_BASE_URL", openaiBaseURL).Msg("get env")
+	openaiAPIKey := os.Getenv("OPENAI_API_KEY")
+	if openaiAPIKey == "" {
+		return errors.Wrap(code.LLMEnvMissedError, "OPENAI_API_KEY missed")
+	}
+	log.Info().Str("OPENAI_API_KEY", maskAPIKey(openaiAPIKey)).Msg("get env")
+	modelName := os.Getenv("LLM_MODEL_NAME")
+	if modelName == "" {
+		return errors.Wrap(code.LLMEnvMissedError, "LLM_MODEL_NAME missed")
+	}
+	log.Info().Str("LLM_MODEL_NAME", modelName).Msg("get env")
+	return nil
+}
+
+func GetEnvConfig(key string) string {
+	return os.Getenv(key)
+}
+
+func GetEnvConfigInJSON(key string) (map[string]interface{}, error) {
+	value := GetEnvConfig(key)
+	if value == "" {
+		return nil, nil
+	}
+
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(value), &result); err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func GetEnvConfigInBool(key string) bool {
+	value := GetEnvConfig(key)
+	if value == "" {
+		return false
+	}
+
+	boolValue, _ := strconv.ParseBool(value)
+	return boolValue
+}
+
+// GetEnvConfigOrDefault get env config or default value
+func GetEnvConfigOrDefault(key, defaultValue string) string {
+	value := GetEnvConfig(key)
+	if value == "" {
+		return defaultValue
+	}
+	return value
+}
+
+func GetEnvConfigInInt(key string, defaultValue int) int {
+	value := GetEnvConfig(key)
+	if value == "" {
+		return defaultValue
+	}
+
+	intValue, err := strconv.Atoi(value)
+	if err != nil {
+		return defaultValue
+	}
+	return intValue
+}
+
+// CustomTransport is a custom RoundTripper that adds headers to every request
+type CustomTransport struct {
+	Transport http.RoundTripper
+	Headers   map[string]string
+}
+
+// RoundTrip executes a single HTTP transaction and adds custom headers
+func (c *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	for key, value := range c.Headers {
+		req.Header.Set(key, value)
+	}
+	return c.Transport.RoundTrip(req)
+}
+
+type OutputFormat struct {
+	Thought string `json:"thought"`
+	Action  string `json:"action"`
+	Error   string `json:"error,omitempty"`
+}
+
+// GetModelConfig get OpenAI config
+func GetModelConfig() (*openai.ChatModelConfig, error) {
+	loadEnv()
+	envConfig := &OpenAIInitConfig{
+		Headers: make(map[string]string),
+	}
+
+	// read from JSON config first
+	jsonStr := GetEnvConfig(EnvOpenAIInitConfigJSON)
+	if jsonStr != "" {
+		if err := json.Unmarshal([]byte(jsonStr), envConfig); err != nil {
+			return nil, err
+		}
+	}
+
+	// outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
+	// if err != nil {
+	// 	log.Fatal().Err(err).Msg("NewSchemaRefForValue failed")
+	// }
+
+	config := &openai.ChatModelConfig{
+		HTTPClient: &http.Client{
+			Timeout: defaultTimeout,
+			Transport: &CustomTransport{
+				Transport: http.DefaultTransport,
+				Headers:   envConfig.Headers,
+			},
+		},
+		// TODO: set structured response format
+		// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
+		// ResponseFormat: &openai2.ChatCompletionResponseFormat{
+		// 	Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
+		// 	JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
+		// 		Name:        "thought_and_action",
+		// 		Description: "data that describes planning thought and action",
+		// 		Schema:      outputFormatSchema.Value,
+		// 		Strict:      false,
+		// 	},
+		// },
+	}
+
+	if baseURL := GetEnvConfig(EnvOpenAIBaseURL); baseURL != "" {
+		config.BaseURL = baseURL
+	} else {
+		return nil, fmt.Errorf("miss env %s", EnvOpenAIBaseURL)
+	}
+
+	if apiKey := GetEnvConfig(EnvOpenAIAPIKey); apiKey != "" {
+		config.APIKey = apiKey
+	} else {
+		return nil, fmt.Errorf("miss env %s", EnvOpenAIAPIKey)
+	}
+
+	if modelName := GetEnvConfig(EnvModelName); modelName != "" {
+		config.Model = modelName
+	} else {
+		return nil, fmt.Errorf("miss env %s", EnvModelName)
+	}
+
+	// log config info
+	log.Info().Str("model", config.Model).
+		Str("baseURL", config.BaseURL).
+		Str("apiKey", maskAPIKey(config.APIKey)).
+		Str("timeout", defaultTimeout.String()).
+		Msg("get model config")
+
+	return config, nil
+}
+
+// maskAPIKey masks the API key
+func maskAPIKey(key string) string {
+	if len(key) <= 8 {
+		return "******"
+	}
+
+	return key[:4] + "******" + key[len(key)-4:]
+}

uixt/ai/llm.go

@@ -1,9 +1,12 @@
 package ai
 
-import "context"
+import (
+	"github.com/cloudwego/eino/schema"
+	"github.com/httprunner/httprunner/v5/uixt/types"
+)
 
 type ILLMService interface {
-	Call(ctx context.Context, prompt string) (string, error)
+	Call(opts *PlanningOptions) (*PlanningResult, error)
 }
 
 func NewGPT4oLLMService() (*openaiLLMService, error) {
@@ -12,6 +15,46 @@ func NewGPT4oLLMService() (*openaiLLMService, error) {
 
 type openaiLLMService struct{}
 
-func (s openaiLLMService) Call(ctx context.Context, prompt string) (string, error) {
+func (s openaiLLMService) Call(opts *PlanningOptions) (*PlanningResult, error) {
-	return "", nil
+	return nil, nil
 }
+
+// PlanningOptions represents the input options for planning
+type PlanningOptions struct {
+	UserInstruction string          `json:"user_instruction"` // append to system prompt
+	Message         *schema.Message `json:"message"`
+	Size            types.Size      `json:"size"`
+}
+
+// PlanningResult represents the result of planning
+type PlanningResult struct {
+	NextActions   []ParsedAction `json:"actions"`
+	ActionSummary string         `json:"summary"`
+}
+
+// VLMResponse represents the response from the Vision Language Model
+type VLMResponse struct {
+	Actions []ParsedAction `json:"actions"`
+	Error   string         `json:"error,omitempty"`
+}
+
+// ParsedAction represents a parsed action from the VLM response
+type ParsedAction struct {
+	ActionType   ActionType             `json:"actionType"`
+	ActionInputs map[string]interface{} `json:"actionInputs"`
+	Thought      string                 `json:"thought"`
+}
+
+type ActionType string
+
+const (
+	ActionTypeClick    ActionType = "click"
+	ActionTypeTap      ActionType = "tap"
+	ActionTypeDrag     ActionType = "drag"
+	ActionTypeSwipe    ActionType = "swipe"
+	ActionTypeWait     ActionType = "wait"
+	ActionTypeFinished ActionType = "finished"
+	ActionTypeCallUser ActionType = "call_user"
+	ActionTypeType     ActionType = "type"
+	ActionTypeScroll   ActionType = "scroll"
+)

uixt/ai/parser.go

@@ -0,0 +1,236 @@
+package ai
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// NewActionParser creates a new ActionParser instance
+func NewActionParser(factor float64) *ActionParser {
+	return &ActionParser{
+		Factor: factor,
+	}
+}
+
+// ActionParser parses VLM responses and converts them to structured actions
+type ActionParser struct {
+	Factor float64 // TODO
+}
+
+// Parse parses the prediction text and extracts actions
+func (p *ActionParser) Parse(predictionText string) ([]ParsedAction, error) {
+	// try parsing JSON format, from VLM like openai/gpt-4o
+	var jsonActions []ParsedAction
+	jsonActions, jsonErr := p.parseJSON(predictionText)
+	if jsonErr == nil {
+		return jsonActions, nil
+	}
+
+	// json parsing failed, try parsing Thought/Action format, from VLM like UI-TARS
+	thoughtActions, thoughtErr := p.parseThoughtAction(predictionText)
+	if thoughtErr == nil {
+		return thoughtActions, nil
+	}
+
+	return nil, errors.Wrap(thoughtErr, "parse planner response failed")
+}
+
+// parseJSON tries to parse the response as JSON format
+func (p *ActionParser) parseJSON(predictionText string) ([]ParsedAction, error) {
+	predictionText = strings.TrimSpace(predictionText)
+	if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
+		predictionText = strings.TrimPrefix(predictionText, "```json")
+		predictionText = strings.TrimSuffix(predictionText, "```")
+	}
+	predictionText = strings.TrimSpace(predictionText)
+
+	var response VLMResponse
+	if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
+		return nil, fmt.Errorf("failed to parse VLM response: %v", err)
+	}
+
+	if response.Error != "" {
+		return nil, errors.New(response.Error)
+	}
+
+	if len(response.Actions) == 0 {
+		return nil, errors.New("no actions returned from VLM")
+	}
+
+	// normalize actions
+	var normalizedActions []ParsedAction
+	for i := range response.Actions {
+		// create a new variable, avoid implicit memory aliasing in for loop.
+		action := response.Actions[i]
+		if err := p.normalizeAction(&action); err != nil {
+			return nil, errors.Wrap(err, "failed to normalize action")
+		}
+		normalizedActions = append(normalizedActions, action)
+	}
+
+	return normalizedActions, nil
+}
+
+// parseThoughtAction parses the Thought/Action format response
+func (p *ActionParser) parseThoughtAction(predictionText string) ([]ParsedAction, error) {
+	thoughtRegex := regexp.MustCompile(`(?is)Thought:(.+?)Action:`)
+	actionRegex := regexp.MustCompile(`(?is)Action:(.+)`)
+
+	// extract Thought part
+	thoughtMatch := thoughtRegex.FindStringSubmatch(predictionText)
+	var thought string
+	if len(thoughtMatch) > 1 {
+		thought = strings.TrimSpace(thoughtMatch[1])
+	}
+
+	// extract Action part, e.g. "click(start_box='(552,454)')"
+	actionMatch := actionRegex.FindStringSubmatch(predictionText)
+	if len(actionMatch) < 2 {
+		return nil, errors.New("no action found in the response")
+	}
+
+	actionText := strings.TrimSpace(actionMatch[1])
+
+	// parse action type and parameters
+	return p.parseActionText(actionText, thought)
+}
+
+// parseActionText parses the action text to extract the action type and parameters
+func (p *ActionParser) parseActionText(actionText, thought string) ([]ParsedAction, error) {
+	// remove trailing comments
+	if idx := strings.Index(actionText, "#"); idx > 0 {
+		actionText = strings.TrimSpace(actionText[:idx])
+	}
+
+	// supported action types and regexes
+	actionRegexes := map[ActionType]*regexp.Regexp{
+		"click":        regexp.MustCompile(`click\(start_box='([^']+)'\)`),
+		"left_double":  regexp.MustCompile(`left_double\(start_box='([^']+)'\)`),
+		"right_single": regexp.MustCompile(`right_single\(start_box='([^']+)'\)`),
+		"drag":         regexp.MustCompile(`drag\(start_box='([^']+)', end_box='([^']+)'\)`),
+		"type":         regexp.MustCompile(`type\(content='([^']+)'\)`),
+		"scroll":       regexp.MustCompile(`scroll\(start_box='([^']+)', direction='([^']+)'\)`),
+		"wait":         regexp.MustCompile(`wait\(\)`),
+		"finished":     regexp.MustCompile(`finished\(content='([^']+)'\)`),
+		"call_user":    regexp.MustCompile(`call_user\(\)`),
+	}
+
+	parsedActions := make([]ParsedAction, 0)
+	for actionType, regex := range actionRegexes {
+		matches := regex.FindStringSubmatch(actionText)
+		if len(matches) == 0 {
+			continue
+		}
+
+		var action ParsedAction
+		action.ActionType = actionType
+		action.ActionInputs = make(map[string]interface{})
+		action.Thought = thought
+
+		// parse parameters based on action type
+		switch actionType {
+		case ActionTypeClick:
+			if len(matches) > 1 {
+				coord, err := p.normalizeCoordinates(matches[1])
+				if err != nil {
+					return nil, errors.Wrapf(err, "normalize point failed: %s", matches[1])
+				}
+				action.ActionInputs["startBox"] = coord
+			}
+		case ActionTypeDrag:
+			if len(matches) > 2 {
+				// handle start point
+				startBox, err := p.normalizeCoordinates(matches[1])
+				if err != nil {
+					return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
+				}
+				action.ActionInputs["startBox"] = startBox
+
+				// handle end point
+				endBox, err := p.normalizeCoordinates(matches[2])
+				if err != nil {
+					return nil, errors.Wrapf(err, "normalize endBox failed: %s", matches[2])
+				}
+				action.ActionInputs["endBox"] = endBox
+			}
+		case ActionTypeType:
+			if len(matches) > 1 {
+				action.ActionInputs["content"] = matches[1]
+			}
+		case ActionTypeScroll:
+			if len(matches) > 2 {
+				startBox, err := p.normalizeCoordinates(matches[1])
+				if err != nil {
+					return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
+				}
+				action.ActionInputs["startBox"] = startBox
+				action.ActionInputs["direction"] = matches[2]
+			}
+		case ActionTypeWait, ActionTypeFinished, ActionTypeCallUser:
+			// 这些动作没有额外参数
+		}
+
+		parsedActions = append(parsedActions, action)
+	}
+
+	if len(parsedActions) == 0 {
+		return nil, fmt.Errorf("no valid actions returned from VLM")
+	}
+	return parsedActions, nil
+}
+
+// normalizeAction normalizes the coordinates in the action
+func (p *ActionParser) normalizeAction(action *ParsedAction) error {
+	switch action.ActionType {
+	case "click", "drag":
+		// handle click and drag action coordinates
+		if startBox, ok := action.ActionInputs["startBox"].(string); ok {
+			normalized, err := p.normalizeCoordinates(startBox)
+			if err != nil {
+				return fmt.Errorf("failed to normalize startBox: %w", err)
+			}
+			action.ActionInputs["startBox"] = normalized
+		}
+
+		if endBox, ok := action.ActionInputs["endBox"].(string); ok {
+			normalized, err := p.normalizeCoordinates(endBox)
+			if err != nil {
+				return fmt.Errorf("failed to normalize endBox: %w", err)
+			}
+			action.ActionInputs["endBox"] = normalized
+		}
+	}
+
+	return nil
+}
+
+// normalizeCoordinates normalizes the coordinates based on the factor
+func (p *ActionParser) normalizeCoordinates(coordStr string) (coords []float64, err error) {
+	// check empty string
+	if coordStr == "" {
+		return nil, fmt.Errorf("empty coordinate string")
+	}
+
+	if !strings.Contains(coordStr, ",") {
+		return nil, fmt.Errorf("invalid coordinate string: %s", coordStr)
+	}
+
+	// remove possible brackets and split coordinates
+	coordStr = strings.Trim(coordStr, "[]() \t")
+
+	// try parsing JSON array
+	jsonStr := coordStr
+	if !strings.HasPrefix(jsonStr, "[") {
+		jsonStr = "[" + coordStr + "]"
+	}
+
+	err = json.Unmarshal([]byte(jsonStr), &coords)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse coordinate string: %w", err)
+	}
+	return coords, nil
+}

uixt/ai/planner.go

@@ -0,0 +1,456 @@
+package ai
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"image"
+	"image/color"
+	"image/draw"
+	_ "image/jpeg"
+	"image/png"
+	"math"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/cloudwego/eino-ext/components/model/openai"
+	"github.com/cloudwego/eino/components/model"
+	"github.com/cloudwego/eino/schema"
+	"github.com/httprunner/httprunner/v5/uixt/types"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+// Error types
+var (
+	ErrEmptyInstruction      = fmt.Errorf("user instruction is empty")
+	ErrNoConversationHistory = fmt.Errorf("conversation history is empty")
+	ErrInvalidImageData      = fmt.Errorf("invalid image data")
+)
+
+func NewPlanner(ctx context.Context) (*Planner, error) {
+	config, err := GetModelConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
+	}
+	model, err := openai.NewChatModel(ctx, config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
+	}
+	parser := NewActionParser(1000)
+	return &Planner{
+		ctx:    ctx,
+		config: config,
+		model:  model,
+		parser: parser,
+	}, nil
+}
+
+type Planner struct {
+	ctx     context.Context
+	model   model.ChatModel
+	config  *openai.ChatModelConfig
+	parser  *ActionParser
+	history []*schema.Message // conversation history
+}
+
+// Call performs UI planning using Vision Language Model
+func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
+	// validate input parameters
+	if err := validateInput(opts); err != nil {
+		return nil, errors.Wrap(err, "validate input parameters failed")
+	}
+
+	// prepare prompt
+	if len(p.history) == 0 {
+		// add system message
+		systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
+		p.history = []*schema.Message{
+			{
+				Role:    schema.System,
+				Content: systemPrompt,
+			},
+		}
+	}
+	// append user image message
+	p.appendConversationHistory(opts.Message)
+
+	// call model service, generate response
+	logRequest(p.history)
+	startTime := time.Now()
+	resp, err := p.model.Generate(p.ctx, p.history)
+	log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
+		Str("model", p.config.Model).Msg("call model service")
+	if err != nil {
+		return nil, fmt.Errorf("request model service failed: %w", err)
+	}
+	logResponse(resp)
+
+	// parse result
+	result, err := p.parseResult(resp, opts.Size)
+	if err != nil {
+		return nil, errors.Wrap(err, "parse result failed")
+	}
+
+	// append assistant message
+	p.appendConversationHistory(&schema.Message{
+		Role:    schema.Assistant,
+		Content: result.ActionSummary,
+	})
+
+	return result, nil
+}
+
+func validateInput(opts *PlanningOptions) error {
+	if opts.UserInstruction == "" {
+		return ErrEmptyInstruction
+	}
+
+	if opts.Message == nil {
+		return ErrNoConversationHistory
+	}
+
+	if opts.Message.Role == schema.User {
+		// check MultiContent
+		if len(opts.Message.MultiContent) > 0 {
+			for _, content := range opts.Message.MultiContent {
+				if content.Type == schema.ChatMessagePartTypeImageURL && content.ImageURL == nil {
+					return ErrInvalidImageData
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func logRequest(messages []*schema.Message) {
+	msgs := make([]*schema.Message, 0, len(messages))
+	for _, message := range messages {
+		msg := &schema.Message{
+			Role: message.Role,
+		}
+		if message.Content != "" {
+			msg.Content = message.Content
+		} else if len(message.MultiContent) > 0 {
+			for _, mc := range message.MultiContent {
+				switch mc.Type {
+				case schema.ChatMessagePartTypeImageURL:
+					// Create a copy of the ImageURL to avoid modifying the original message
+					imageURLCopy := *mc.ImageURL
+					if strings.HasPrefix(imageURLCopy.URL, "data:image/") {
+						imageURLCopy.URL = "<data:image/base64...>"
+					}
+					msg.MultiContent = append(msg.MultiContent, schema.ChatMessagePart{
+						Type:     mc.Type,
+						ImageURL: &imageURLCopy,
+					})
+				}
+			}
+		}
+		msgs = append(msgs, msg)
+	}
+	log.Debug().Interface("messages", msgs).Msg("log request messages")
+}
+
+func logResponse(resp *schema.Message) {
+	logger := log.Info().Str("role", string(resp.Role)).
+		Str("content", resp.Content)
+	if resp.ResponseMeta != nil {
+		logger = logger.Interface("response_meta", resp.ResponseMeta)
+	}
+	if resp.Extra != nil {
+		logger = logger.Interface("extra", resp.Extra)
+	}
+	logger.Msg("log response message")
+}
+
+// appendConversationHistory adds a message to the conversation history
+func (p *Planner) appendConversationHistory(msg *schema.Message) {
+	// for user image message:
+	// - keep at most 4 user image messages
+	// - delete the oldest user image message when the limit is reached
+	if msg.Role == schema.User {
+		// get all existing user messages
+		userImgCount := 0
+		firstUserImgIndex := -1
+
+		// calculate the number of user messages and find the index of the first user message
+		for i, item := range p.history {
+			if item.Role == schema.User {
+				userImgCount++
+				if firstUserImgIndex == -1 {
+					firstUserImgIndex = i
+				}
+			}
+		}
+
+		// if there are already 4 user messages, delete the first one before adding the new message
+		if userImgCount >= 4 && firstUserImgIndex >= 0 {
+			// delete the first user message
+			p.history = append(
+				p.history[:firstUserImgIndex],
+				p.history[firstUserImgIndex+1:]...,
+			)
+		}
+		// add the new user message to the history
+		p.history = append(p.history, msg)
+	}
+
+	// for assistant message:
+	// - keep at most the last 10 assistant messages
+	if msg.Role == schema.Assistant {
+		// add the new assistant message to the history
+		p.history = append(p.history, msg)
+
+		// if there are more than 10 assistant messages, remove the oldest ones
+		assistantMsgCount := 0
+		for i := len(p.history) - 1; i >= 0; i-- {
+			if p.history[i].Role == schema.Assistant {
+				assistantMsgCount++
+				if assistantMsgCount > 10 {
+					p.history = append(p.history[:i], p.history[i+1:]...)
+				}
+			}
+		}
+	}
+}
+
+func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
+	// parse response
+	parseActions, err := p.parser.Parse(msg.Content)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse actions: %w", err)
+	}
+
+	// process response
+	result, err := processVLMResponse(parseActions, size)
+	if err != nil {
+		return nil, errors.Wrap(err, "process VLM response failed")
+	}
+
+	log.Info().
+		Interface("summary", result.ActionSummary).
+		Interface("actions", result.NextActions).
+		Msg("get VLM planning result")
+	return result, nil
+}
+
+// processVLMResponse processes the VLM response and converts it to PlanningResult
+func processVLMResponse(actions []ParsedAction, size types.Size) (*PlanningResult, error) {
+	log.Info().Msg("processing VLM response...")
+
+	if len(actions) == 0 {
+		return nil, fmt.Errorf("no actions returned from VLM")
+	}
+
+	// validate and post-process each action
+	for i := range actions {
+		// validate action type
+		switch actions[i].ActionType {
+		case "click":
+			if err := convertCoordinateAction(&actions[i], "startBox", size); err != nil {
+				return nil, errors.Wrap(err, "convert coordinate action failed")
+			}
+		case "drag":
+			if err := convertCoordinateAction(&actions[i], "startBox", size); err != nil {
+				return nil, errors.Wrap(err, "convert coordinate action failed")
+			}
+			if err := convertCoordinateAction(&actions[i], "endBox", size); err != nil {
+				return nil, errors.Wrap(err, "convert coordinate action failed")
+			}
+		case "type":
+			validateTypeContent(&actions[i])
+		case "wait", "finished", "call_user":
+			// these actions do not need extra parameters
+		default:
+			log.Printf("warning: unknown action type: %s, will try to continue processing", actions[i].ActionType)
+		}
+	}
+
+	// extract action summary
+	actionSummary := extractActionSummary(actions)
+
+	return &PlanningResult{
+		NextActions:   actions,
+		ActionSummary: actionSummary,
+	}, nil
+}
+
+// extractActionSummary extracts the summary from the actions
+func extractActionSummary(actions []ParsedAction) string {
+	if len(actions) == 0 {
+		return ""
+	}
+
+	// use the Thought of the first action as summary
+	if actions[0].Thought != "" {
+		return actions[0].Thought
+	}
+
+	// if no Thought, generate summary from action type
+	action := actions[0]
+	switch action.ActionType {
+	case "click":
+		return "点击操作"
+	case "drag":
+		return "拖拽操作"
+	case "type":
+		content, _ := action.ActionInputs["content"].(string)
+		if len(content) > 20 {
+			content = content[:20] + "..."
+		}
+		return fmt.Sprintf("输入文本: %s", content)
+	case "wait":
+		return "等待操作"
+	case "finished":
+		return "完成操作"
+	case "call_user":
+		return "请求用户协助"
+	default:
+		return fmt.Sprintf("执行 %s 操作", action.ActionType)
+	}
+}
+
+func convertCoordinateAction(action *ParsedAction, boxField string, size types.Size) error {
+	// The model generates a 2D coordinate output that represents relative positions.
+	// To convert these values to image-relative coordinates, divide each component by 1000 to obtain values in the range [0,1].
+	// The absolute coordinates required by the Action can be calculated by:
+	// - X absolute = X relative × image width / 1000
+	// - Y absolute = Y relative × image height / 1000
+
+	// get image width and height
+	imageWidth := size.Width
+	imageHeight := size.Height
+
+	box := action.ActionInputs[boxField]
+	coords, ok := box.([]float64)
+	if !ok {
+		log.Error().Interface("inputs", action.ActionInputs).Msg("invalid action inputs")
+		return fmt.Errorf("invalid action inputs")
+	}
+
+	if len(coords) == 2 {
+		coords[0] = math.Round((coords[0]/1000*float64(imageWidth))*10) / 10
+		coords[1] = math.Round((coords[1]/1000*float64(imageHeight))*10) / 10
+	} else if len(coords) == 4 {
+		coords[0] = math.Round((coords[0]/1000*float64(imageWidth))*10) / 10
+		coords[1] = math.Round((coords[1]/1000*float64(imageHeight))*10) / 10
+		coords[2] = math.Round((coords[2]/1000*float64(imageWidth))*10) / 10
+		coords[3] = math.Round((coords[3]/1000*float64(imageHeight))*10) / 10
+	} else {
+		log.Error().Interface("inputs", action.ActionInputs).Msg("invalid action inputs")
+		return fmt.Errorf("invalid action inputs")
+	}
+
+	return nil
+}
+
+// validateTypeContent 验证输入文本内容
+func validateTypeContent(action *ParsedAction) {
+	if content, ok := action.ActionInputs["content"]; !ok || content == "" {
+		// default to empty string
+		action.ActionInputs["content"] = ""
+		log.Warn().Msg("type action missing content parameter, set to default")
+	}
+}
+
+// SavePositionImg saves an image with position markers
+func SavePositionImg(params struct {
+	InputImgBase64 string
+	Rect           struct {
+		X float64
+		Y float64
+	}
+	OutputPath string
+}) error {
+	// 解码Base64图像
+	imgData := params.InputImgBase64
+	// 如果包含了数据URL前缀，去掉它
+	if strings.HasPrefix(imgData, "data:image/") {
+		parts := strings.Split(imgData, ",")
+		if len(parts) > 1 {
+			imgData = parts[1]
+		}
+	}
+
+	// 解码Base64
+	unbased, err := base64.StdEncoding.DecodeString(imgData)
+	if err != nil {
+		return fmt.Errorf("无法解码Base64图像: %w", err)
+	}
+
+	// 解码图像
+	reader := bytes.NewReader(unbased)
+	img, _, err := image.Decode(reader)
+	if err != nil {
+		return fmt.Errorf("无法解码图像数据: %w", err)
+	}
+
+	// 创建一个可以在其上绘制的图像
+	bounds := img.Bounds()
+	rgba := image.NewRGBA(bounds)
+	draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
+
+	// 在点击/拖动位置绘制标记
+	markRadius := 10
+	x, y := int(params.Rect.X), int(params.Rect.Y)
+
+	// 绘制红色圆圈
+	for i := -markRadius; i <= markRadius; i++ {
+		for j := -markRadius; j <= markRadius; j++ {
+			if i*i+j*j <= markRadius*markRadius {
+				if x+i >= 0 && x+i < bounds.Max.X && y+j >= 0 && y+j < bounds.Max.Y {
+					rgba.Set(x+i, y+j, color.RGBA{255, 0, 0, 255})
+				}
+			}
+		}
+	}
+
+	// 保存图像
+	outFile, err := os.Create(params.OutputPath)
+	if err != nil {
+		return fmt.Errorf("无法创建输出文件: %w", err)
+	}
+	defer outFile.Close()
+
+	// 编码为PNG并保存
+	if err := png.Encode(outFile, rgba); err != nil {
+		return fmt.Errorf("无法编码和保存图像: %w", err)
+	}
+
+	return nil
+}
+
+// loadImage loads image and returns base64 encoded string
+func loadImage(imagePath string) (base64Str string, size types.Size, err error) {
+	// Read the image file
+	imageFile, err := os.Open(imagePath)
+	if err != nil {
+		return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
+	}
+	defer imageFile.Close()
+
+	// Decode the image to get its resolution
+	imageData, format, err := image.Decode(imageFile)
+	if err != nil {
+		return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	// Get the resolution of the image
+	width := imageData.Bounds().Dx()
+	height := imageData.Bounds().Dy()
+	size = types.Size{Width: width, Height: height}
+
+	// Convert image to base64
+	buf := new(bytes.Buffer)
+	if err := png.Encode(buf, imageData); err != nil {
+		return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
+	}
+	base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
+		base64.StdEncoding.EncodeToString(buf.Bytes()))
+
+	return base64Str, size, nil
+}

uixt/ai/planner_test.go

@@ -0,0 +1,363 @@
+package ai
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/cloudwego/eino/schema"
+	"github.com/httprunner/httprunner/v5/uixt/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVLMPlanning(t *testing.T) {
+	imageBase64, size, err := loadImage("testdata/llk_1.png")
+	require.NoError(t, err)
+
+	userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
+	1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
+	2. 连接规则:
+	- 两个相同的图案可以通过不超过三条直线连接。
+	- 连接线可以水平或垂直，但不能穿过其他图案。
+	- 连接线的转折次数不能超过两次。
+	3. 游戏界面: 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
+	4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
+	5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
+	6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。`
+
+	userInstruction += "\n\n请基于以上游戏规则，给出下一步可点击的两个图标坐标"
+
+	planner, err := NewPlanner(context.Background())
+	require.NoError(t, err)
+
+	opts := &PlanningOptions{
+		UserInstruction: userInstruction,
+		Message: &schema.Message{
+			Role: schema.User,
+			MultiContent: []schema.ChatMessagePart{
+				{
+					Type: schema.ChatMessagePartTypeImageURL,
+					ImageURL: &schema.ChatMessageImageURL{
+						URL: imageBase64,
+					},
+				},
+			},
+		},
+		Size: size,
+	}
+
+	// 执行规划
+	result, err := planner.Call(opts)
+
+	// 验证结果
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.NotEmpty(t, result.NextActions)
+
+	// 验证动作
+	action := result.NextActions[0]
+	assert.NotEmpty(t, action.ActionType)
+	assert.NotEmpty(t, action.Thought)
+
+	// 根据动作类型验证参数
+	switch action.ActionType {
+	case "click", "drag", "left_double", "right_single", "scroll":
+		// 这些动作需要验证坐标
+		assert.NotEmpty(t, action.ActionInputs["startBox"])
+
+		// 验证坐标格式
+		coords, ok := action.ActionInputs["startBox"].([]float64)
+		require.True(t, ok)
+		require.True(t, len(coords) >= 2) // 至少有 x, y 坐标
+
+		// 验证坐标范围
+		for _, coord := range coords {
+			assert.GreaterOrEqual(t, coord, float64(0))
+		}
+
+	case "type":
+		// 验证文本内容
+		assert.NotEmpty(t, action.ActionInputs["content"])
+
+	case "hotkey":
+		// 验证按键
+		assert.NotEmpty(t, action.ActionInputs["key"])
+
+	case "wait", "finished", "call_user":
+		// 这些动作不需要额外参数
+
+	default:
+		t.Fatalf("未知的动作类型: %s", action.ActionType)
+	}
+}
+
+func TestXHSPlanning(t *testing.T) {
+	imageBase64, size, err := loadImage("testdata/xhs-feed.jpeg")
+	require.NoError(t, err)
+
+	userInstruction := "点击第二个帖子的作者头像"
+
+	planner, err := NewPlanner(context.Background())
+	require.NoError(t, err)
+
+	opts := &PlanningOptions{
+		UserInstruction: userInstruction,
+		Message: &schema.Message{
+			Role: schema.User,
+			MultiContent: []schema.ChatMessagePart{
+				{
+					Type: schema.ChatMessagePartTypeImageURL,
+					ImageURL: &schema.ChatMessageImageURL{
+						URL: imageBase64,
+					},
+				},
+			},
+		},
+		Size: size,
+	}
+
+	// 执行规划
+	result, err := planner.Call(opts)
+
+	// 验证结果
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.NotEmpty(t, result.NextActions)
+
+	// 验证动作
+	action := result.NextActions[0]
+	assert.NotEmpty(t, action.ActionType)
+	assert.NotEmpty(t, action.Thought)
+
+	// 根据动作类型验证参数
+	switch action.ActionType {
+	case "click", "drag", "left_double", "right_single", "scroll":
+		// 这些动作需要验证坐标
+		assert.NotEmpty(t, action.ActionInputs["startBox"])
+
+		// 验证坐标格式
+		coords, ok := action.ActionInputs["startBox"].([]float64)
+		require.True(t, ok)
+		require.True(t, len(coords) >= 2) // 至少有 x, y 坐标
+
+		// 验证坐标范围
+		for _, coord := range coords {
+			assert.GreaterOrEqual(t, coord, float64(0))
+		}
+
+	case "type":
+		// 验证文本内容
+		assert.NotEmpty(t, action.ActionInputs["content"])
+
+	case "hotkey":
+		// 验证按键
+		assert.NotEmpty(t, action.ActionInputs["key"])
+
+	case "wait", "finished", "call_user":
+		// 这些动作不需要额外参数
+
+	default:
+		t.Fatalf("未知的动作类型: %s", action.ActionType)
+	}
+}
+
+func TestChatList(t *testing.T) {
+	imageBase64, size, err := loadImage("testdata/chat_list.jpeg")
+	require.NoError(t, err)
+
+	userInstruction := "请结合图片的文字信息，请告诉我一共有多少个群聊，哪些群聊右下角有绿点"
+
+	planner, err := NewPlanner(context.Background())
+	require.NoError(t, err)
+
+	opts := &PlanningOptions{
+		UserInstruction: userInstruction,
+		Message: &schema.Message{
+			Role: schema.User,
+			MultiContent: []schema.ChatMessagePart{
+				{
+					Type: schema.ChatMessagePartTypeImageURL,
+					ImageURL: &schema.ChatMessageImageURL{
+						URL: imageBase64,
+					},
+				},
+			},
+		},
+		Size: size,
+	}
+
+	// 执行规划
+	result, err := planner.Call(opts)
+
+	// 验证结果
+	require.NoError(t, err)
+	require.NotNil(t, result)
+}
+
+func TestValidateInput(t *testing.T) {
+	imageBase64, size, err := loadImage("testdata/popup_risk_warning.png")
+	require.NoError(t, err)
+
+	tests := []struct {
+		name    string
+		opts    *PlanningOptions
+		wantErr error
+	}{
+		{
+			name: "valid input",
+			opts: &PlanningOptions{
+				UserInstruction: "点击继续使用按钮",
+				Message: &schema.Message{
+					Role: schema.User,
+					MultiContent: []schema.ChatMessagePart{
+						{
+							Type: "image_url",
+							ImageURL: &schema.ChatMessageImageURL{
+								URL: imageBase64,
+							},
+						},
+					},
+				},
+				Size: size,
+			},
+			wantErr: nil,
+		},
+		{
+			name: "empty instruction",
+			opts: &PlanningOptions{
+				UserInstruction: "",
+				Message: &schema.Message{
+					Role:    schema.User,
+					Content: "",
+				},
+				Size: size,
+			},
+			wantErr: ErrEmptyInstruction,
+		},
+		{
+			name: "empty conversation history",
+			opts: &PlanningOptions{
+				UserInstruction: "点击立即卸载按钮",
+				Message:         &schema.Message{},
+			},
+			wantErr: ErrNoConversationHistory,
+		},
+		{
+			name: "invalid image data",
+			opts: &PlanningOptions{
+				UserInstruction: "点击继续使用按钮",
+				Message: &schema.Message{
+					Role:    schema.User,
+					Content: "no image",
+				},
+				Size: size,
+			},
+			wantErr: ErrInvalidImageData,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateInput(tt.opts)
+			if tt.wantErr != nil {
+				assert.Error(t, err)
+				assert.Equal(t, tt.wantErr, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestProcessVLMResponse(t *testing.T) {
+	tests := []struct {
+		name    string
+		actions []ParsedAction
+		wantErr bool
+	}{
+		{
+			name: "valid response",
+			actions: []ParsedAction{
+				{
+					ActionType: "click",
+					ActionInputs: map[string]interface{}{
+						"startBox": []float64{0.5, 0.5},
+					},
+					Thought: "点击中心位置",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name:    "empty actions",
+			actions: []ParsedAction{},
+			wantErr: true,
+		},
+	}
+
+	size := types.Size{
+		Width:  1000,
+		Height: 1000,
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := processVLMResponse(tt.actions, size)
+			if tt.wantErr {
+				assert.Error(t, err)
+				assert.Nil(t, result)
+				return
+			}
+
+			assert.NoError(t, err)
+			assert.NotNil(t, result)
+			assert.Equal(t, tt.actions, result.NextActions)
+		})
+	}
+}
+
+func TestSavePositionImg(t *testing.T) {
+	imageBase64, _, err := loadImage("testdata/popup_risk_warning.png")
+	require.NoError(t, err)
+
+	params := struct {
+		InputImgBase64 string
+		Rect           struct {
+			X float64
+			Y float64
+		}
+		OutputPath string
+	}{
+		InputImgBase64: imageBase64,
+		Rect: struct {
+			X float64
+			Y float64
+		}{
+			X: 100,
+			Y: 100,
+		},
+		OutputPath: "testdata/output.png",
+	}
+
+	err = SavePositionImg(params)
+	require.NoError(t, err)
+
+	// cleanup
+	defer os.Remove(params.OutputPath)
+}
+
+func TestLoadImage(t *testing.T) {
+	// Test PNG image
+	pngBase64, pngSize, err := loadImage("testdata/llk_1.png")
+	require.NoError(t, err)
+	assert.NotEmpty(t, pngBase64)
+	assert.Greater(t, pngSize.Width, 0)
+	assert.Greater(t, pngSize.Height, 0)
+
+	// Test JPEG image
+	jpegBase64, jpegSize, err := loadImage("testdata/xhs-feed.jpeg")
+	require.NoError(t, err)
+	assert.NotEmpty(t, jpegBase64)
+	assert.Greater(t, jpegSize.Width, 0)
+	assert.Greater(t, jpegSize.Height, 0)
+}

uixt/ai/prompt-ui-tars.go

@@ -0,0 +1,24 @@
+package ai
+
+const uiTarsPlanningPrompt = `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+## Output Format
+Thought: ...
+Action: ...
+
+## Action Space
+click(start_box='[x1,y1]')
+long_press(start_box='[x1,y1]', time='')
+type(content='')
+drag(start_box='[x1,y1]', end_box='[x2,y2]')
+press_home()
+press_back()
+finished(content='') # Submit the task regardless of whether it succeeds or fails.
+
+## Note
+- Use Chinese in Thought part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in Thought part.
+
+## User Instruction
+`

uixt/android_driver_adb.go

@@ -944,7 +944,88 @@ func (ad *ADBDriver) OpenUrl(url string) (err error) {
 
 func (ad *ADBDriver) PushImage(localPath string) error {
 	log.Info().Str("localPath", localPath).Msg("ADBDriver.PushImage")
-	remotePath := path.Join("/sdcard/DCIM/Camera/", path.Base(localPath))
+	remoteDir := "/sdcard/DCIM/Camera/"
+	return ad.PushFile(localPath, remoteDir)
+}
+
+// PullImages pulls all images from device's DCIM/Camera directory to local directory
+func (ad *ADBDriver) PullImages(localDir string) error {
+	log.Info().Str("localDir", localDir).Msg("ADBDriver.PullImages")
+	remoteDir := "/sdcard/DCIM/Camera/"
+
+	// create local directory if not exists
+	if err := os.MkdirAll(localDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create local directory: %w", err)
+	}
+
+	files, err := ad.Device.List(remoteDir)
+	if err != nil {
+		return fmt.Errorf("failed to list directory %s: %w", remoteDir, err)
+	}
+
+	for _, file := range files {
+		// filter image files by extension
+		ext := strings.ToLower(path.Ext(file.Name))
+		if !isImageFile(ext) {
+			continue
+		}
+
+		remotePath := path.Join(remoteDir, file.Name)
+		localPath := path.Join(localDir, file.Name)
+
+		// check if file already exists
+		if _, err := os.Stat(localPath); err == nil {
+			log.Debug().Str("localPath", localPath).Msg("file already exists, skipping")
+			continue
+		}
+
+		// create local file
+		f, err := os.Create(localPath)
+		if err != nil {
+			log.Error().Err(err).Str("localPath", localPath).Msg("failed to create local file")
+			continue
+		}
+		defer f.Close()
+
+		// pull image file
+		if err := ad.Device.Pull(remotePath, f); err != nil {
+			log.Error().Err(err).
+				Str("remotePath", remotePath).
+				Str("localPath", localPath).
+				Msg("failed to pull image")
+			continue // continue with next file
+		}
+		log.Info().
+			Str("remotePath", remotePath).
+			Str("localPath", localPath).
+			Msg("image pulled successfully")
+	}
+	return nil
+}
+
+// isImageFile checks if the file extension is an image format
+func isImageFile(ext string) bool {
+	imageExts := map[string]bool{
+		".jpg":  true,
+		".jpeg": true,
+		".png":  true,
+		".gif":  true,
+		".bmp":  true,
+		".webp": true,
+		".heic": true,
+	}
+	return imageExts[ext]
+}
+
+func (ad *ADBDriver) ClearImages() error {
+	log.Info().Msg("ADBDriver.ClearImages")
+	_, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*")
+	return nil
+}
+
+func (ad *ADBDriver) PushFile(localPath string, remoteDir string) error {
+	log.Info().Str("localPath", localPath).Str("remoteDir", remoteDir).Msg("ADBDriver.PushFile")
+	remotePath := path.Join(remoteDir, path.Base(localPath))
 	if err := ad.Device.PushFile(localPath, remotePath); err != nil {
 		return err
 	}
@@ -955,9 +1036,60 @@ func (ad *ADBDriver) PushImage(localPath string) error {
 	return nil
 }
 
-func (ad *ADBDriver) ClearImages() error {
+func (ad *ADBDriver) PullFiles(localDir string, remoteDirs ...string) error {
-	log.Info().Msg("ADBDriver.ClearImages")
+	log.Info().Str("localDir", localDir).Strs("remoteDirs", remoteDirs).Msg("ADBDriver.PullFiles")
-	_, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*")
+
+	// create local directory if not exists
+	if err := os.MkdirAll(localDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create local directory: %w", err)
+	}
+
+	for _, remoteDir := range remoteDirs {
+		files, err := ad.Device.List(remoteDir)
+		if err != nil {
+			return fmt.Errorf("failed to list directory %s: %w", remoteDir, err)
+		}
+
+		for _, file := range files {
+			remotePath := path.Join(remoteDir, file.Name)
+			localPath := path.Join(localDir, file.Name)
+
+			// check if file already exists
+			if _, err := os.Stat(localPath); err == nil {
+				log.Debug().Str("localPath", localPath).Msg("file already exists, skipping")
+				continue
+			}
+
+			// create local file
+			f, err := os.Create(localPath)
+			if err != nil {
+				log.Error().Err(err).Str("localPath", localPath).Msg("failed to create local file")
+				continue
+			}
+			defer f.Close()
+
+			// pull image file
+			if err := ad.Device.Pull(remotePath, f); err != nil {
+				log.Error().Err(err).
+					Str("remotePath", remotePath).
+					Str("localPath", localPath).
+					Msg("failed to pull file")
+				continue // continue with next file
+			}
+			log.Info().
+				Str("remotePath", remotePath).
+				Str("localPath", localPath).
+				Msg("file pulled successfully")
+		}
+	}
+	return nil
+}
+
+func (ad *ADBDriver) ClearFiles(paths ...string) error {
+	log.Info().Strs("paths", paths).Msg("ADBDriver.ClearFiles")
+	for _, path := range paths {
+		_, _ = ad.Device.RunShellCommand("rm", "-rf", path)
+	}
 	return nil
 }
 

uixt/android_test.go

@@ -24,7 +24,9 @@ func setupADBDriverExt(t *testing.T) *XTDriver {
 	driver, err := device.NewDriver()
 	require.Nil(t, err)
 	return NewXTDriver(driver,
-		ai.WithCVService(ai.CVServiceTypeVEDEM))
+		ai.WithCVService(ai.CVServiceTypeVEDEM),
+		ai.WithLLMService(ai.LLMServiceTypeUITARS),
+	)
 }
 
 func setupUIA2DriverExt(t *testing.T) *XTDriver {
@@ -258,6 +260,9 @@ func TestDriver_ADB_PushImage(t *testing.T) {
 	err = driver.PushImage(path)
 	assert.Nil(t, err)
 
+	err = driver.PullImages("./test")
+	assert.Nil(t, err)
+
 	err = driver.ClearImages()
 	assert.Nil(t, err)
 }

uixt/browser_driver.go

@@ -444,11 +444,27 @@ func (wd *BrowserDriver) AppClear(packageName string) error {
 	return errors.New("not support")
 }
 
+func (wd *BrowserDriver) PushImage(localPath string) error {
+	return errors.New("not support")
+}
+
+func (wd *BrowserDriver) PullImages(localDir string) error {
+	return errors.New("not support")
+}
+
 func (wd *BrowserDriver) ClearImages() error {
 	return errors.New("not support")
 }
 
-func (wd *BrowserDriver) PushImage(localPath string) error {
+func (wd *BrowserDriver) PushFile(localPath string, remoteDir string) error {
+	return errors.New("not support")
+}
+
+func (wd *BrowserDriver) PullFiles(localDir string, remoteDirs ...string) error {
+	return errors.New("not support")
+}
+
+func (wd *BrowserDriver) ClearFiles(paths ...string) error {
 	return errors.New("not support")
 }
 

uixt/driver.go

@@ -68,8 +68,14 @@ type IDriver interface {
 
 	// image related
 	PushImage(localPath string) error
+	PullImages(localDir string) error
 	ClearImages() error
 
+	// files related
+	PushFile(localPath string, remoteDir string) error
+	PullFiles(localDir string, remoteDirs ...string) error
+	ClearFiles(paths ...string) error
+
 	// triggers the log capture and returns the log entries
 	StartCaptureLog(identifier ...string) error
 	StopCaptureLog() (result interface{}, err error)

uixt/driver_action.go

@@ -45,6 +45,7 @@ const (
 	ACTION_Input       ActionMethod = "input"
 	ACTION_Back        ActionMethod = "back"
 	ACTION_KeyCode     ActionMethod = "keycode"
+	ACTION_AIAction    ActionMethod = "ai_action" // action with ai
 
 	// custom actions
 	ACTION_SwipeToTapApp   ActionMethod = "swipe_to_tap_app"   // swipe left & right to find app and tap
@@ -64,12 +65,14 @@ const (
 	SelectorLabel         string = "ui_label"
 	SelectorOCR           string = "ui_ocr"
 	SelectorImage         string = "ui_image"
+	SelectorAI            string = "ui_ai" // ui query with ai
 	SelectorForegroundApp string = "ui_foreground_app"
 	// assertions
 	AssertionEqual     string = "equal"
 	AssertionNotEqual  string = "not_equal"
 	AssertionExists    string = "exists"
 	AssertionNotExists string = "not_exists"
+	AssertionAI        string = "ai_assert" // assert with ai
 )
 
 type MobileAction struct {
@@ -281,6 +284,11 @@ func (dExt *XTDriver) DoAction(action MobileAction) (err error) {
 		fn := action.Fn
 		fn()
 		return nil
+	case ACTION_AIAction:
+		if prompt, ok := action.Params.(string); ok {
+			return dExt.AIAction(prompt, action.GetOptions()...)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_AIAction, action.Params)
 	default:
 		log.Warn().Str("action", string(action.Method)).Msg("action not implemented")
 		return errors.Wrapf(code.InvalidCaseError,

uixt/driver_ext_ai.go

@@ -0,0 +1,118 @@
+package uixt
+
+import (
+	"encoding/base64"
+	"fmt"
+	"path/filepath"
+
+	"github.com/cloudwego/eino/schema"
+	"github.com/httprunner/httprunner/v5/code"
+	"github.com/httprunner/httprunner/v5/internal/builtin"
+	"github.com/httprunner/httprunner/v5/internal/config"
+	"github.com/httprunner/httprunner/v5/uixt/ai"
+	"github.com/httprunner/httprunner/v5/uixt/option"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+func (dExt *XTDriver) StartToGoal(text string, opts ...option.ActionOption) error {
+	options := option.NewActionOptions(opts...)
+	var attempt int
+	for {
+		attempt++
+		log.Info().Int("attempt", attempt).Msg("planning attempt")
+		if err := dExt.AIAction(text, opts...); err != nil {
+			return err
+		}
+
+		if options.MaxRetryTimes > 1 && attempt >= options.MaxRetryTimes {
+			return errors.New("reached max retry times")
+		}
+	}
+}
+
+func (dExt *XTDriver) AIAction(text string, opts ...option.ActionOption) error {
+	// plan next action
+	result, err := dExt.PlanNextAction(text, opts...)
+	if err != nil {
+		return err
+	}
+
+	// do actions
+	for _, action := range result.NextActions {
+		switch action.ActionType {
+		case ai.ActionTypeClick:
+			point := action.ActionInputs["startBox"].([]float64)
+			if err := dExt.TapAbsXY(point[0], point[1], opts...); err != nil {
+				return err
+			}
+		case ai.ActionTypeFinished:
+			log.Info().Msg("ai action done")
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (string, error) {
+	return "", nil
+}
+
+func (dExt *XTDriver) AIAssert(text string, opts ...option.ActionOption) error {
+	return nil
+}
+
+func (dExt *XTDriver) PlanNextAction(text string, opts ...option.ActionOption) (*ai.PlanningResult, error) {
+	if dExt.LLMService == nil {
+		return nil, errors.New("LLM service is not initialized")
+	}
+
+	compressedBufSource, err := dExt.GetScreenShotBuffer()
+	if err != nil {
+		return nil, err
+	}
+
+	// convert buffer to base64 string
+	screenShotBase64 := "data:image/jpeg;base64," +
+		base64.StdEncoding.EncodeToString(compressedBufSource.Bytes())
+
+	// save screenshot to file
+	imagePath := filepath.Join(
+		config.GetConfig().ScreenShotsPath,
+		fmt.Sprintf("%s.jpeg", builtin.GenNameWithTimestamp("%d_screenshot")),
+	)
+	go func() {
+		err := saveScreenShot(compressedBufSource, imagePath)
+		if err != nil {
+			log.Error().Err(err).Msg("save screenshot file failed")
+		}
+	}()
+
+	size, err := dExt.IDriver.WindowSize()
+	if err != nil {
+		return nil, errors.Wrap(code.DeviceGetInfoError, err.Error())
+	}
+
+	planningOpts := &ai.PlanningOptions{
+		UserInstruction: text,
+		Message: &schema.Message{
+			Role: schema.User,
+			MultiContent: []schema.ChatMessagePart{
+				{
+					Type: schema.ChatMessagePartTypeImageURL,
+					ImageURL: &schema.ChatMessageImageURL{
+						URL: screenShotBase64,
+					},
+				},
+			},
+		},
+		Size: size,
+	}
+
+	result, err := dExt.LLMService.Call(planningOpts)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get next action from planner")
+	}
+	return result, nil
+}

uixt/driver_ext_screenshot.go

@@ -7,7 +7,6 @@ import (
 	"image/gif"
 	"image/jpeg"
 	"image/png"
-	_ "image/png"
 	"os"
 	"path/filepath"
 	"strings"
@@ -45,10 +44,7 @@ func (s *ScreenResult) FilterTextsByScope(x1, y1, x2, y2 float64) ai.OCRTexts {
 	})
 }
 
-// GetScreenResult takes a screenshot, returns the image recognition result
+func (dExt *XTDriver) GetScreenShotBuffer() (compressedBufSource *bytes.Buffer, err error) {
-func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult *ScreenResult, err error) {
-	screenshotOptions := option.NewActionOptions(opts...)
-
 	// take screenshot
 	bufSource, err := dExt.ScreenShot()
 	if err != nil {
@@ -63,6 +59,19 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
 			"compress screenshot failed %v", err)
 	}
 
+	return compressBufSource, nil
+}
+
+// GetScreenResult takes a screenshot, returns the image recognition result
+func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult *ScreenResult, err error) {
+	// get compressed screenshot buffer
+	compressBufSource, err := dExt.GetScreenShotBuffer()
+	if err != nil {
+		return nil, err
+	}
+
+	screenshotOptions := option.NewActionOptions(opts...)
+
 	// save compressed screenshot to file
 	var fileName string
 	optionsList := screenshotOptions.List()
@@ -81,8 +90,6 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
 		err := saveScreenShot(compressBufSource, imagePath)
 		if err != nil {
 			log.Error().Err(err).Msg("save screenshot file failed")
-		} else {
-			log.Info().Str("path", imagePath).Msg("screenshot saved")
 		}
 	}()
 

uixt/driver_ext_test.go

@@ -123,6 +123,47 @@ func TestDriverExt_TapByOCR(t *testing.T) {
 	assert.Nil(t, err)
 }
 
+func TestDriverExt_TapByLLM(t *testing.T) {
+	driver := setupDriverExt(t)
+	err := driver.AIAction("点击第一个帖子的作者头像")
+	assert.Nil(t, err)
+}
+
+func TestDriverExt_StartToGoal(t *testing.T) {
+	driver := setupDriverExt(t)
+
+	userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
+	1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
+	2. 连接规则:
+	- 两个相同的图案可以通过不超过三条直线连接。
+	- 连接线可以水平或垂直，但不能斜线，也不能跨过其他图案。
+	- 连接线的转折次数不能超过两次。
+	3. 游戏界面:
+	- 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
+	- 图案或图标在未选中状态下背景为白色，选中状态下背景为绿色。
+	4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
+	5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
+	6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。
+
+	注意事项：
+	1、当连接错误时，顶部的红心会减少一个，需及时调整策略，避免红心变为0个后游戏失败
+	2、不要连续 2 次点击同一个图案
+	3、不要犯重复的错误
+	`
+
+	userInstruction += "\n\n请严格按照以上游戏规则，开始游戏；注意，请只做点击操作"
+
+	err := driver.StartToGoal(userInstruction)
+	assert.Nil(t, err)
+}
+
+func TestDriverExt_PlanNextAction(t *testing.T) {
+	driver := setupDriverExt(t)
+	result, err := driver.PlanNextAction("启动抖音")
+	assert.Nil(t, err)
+	t.Log(result)
+}
+
 func TestDriverExt_prepareSwipeAction(t *testing.T) {
 	driver := setupDriverExt(t)
 

uixt/driver_utils.go

@@ -185,8 +185,12 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st
 	switch check {
 	case SelectorOCR:
 		err = dExt.assertOCR(expected, assert)
+	// case SelectorAI:
+	// 	// TODO
 	case SelectorForegroundApp:
 		err = dExt.assertForegroundApp(expected, assert)
+	default:
+		return fmt.Errorf("validator %s not implemented", check)
 	}
 
 	if err != nil {
@@ -198,7 +202,8 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st
 		return err
 	}
 
-	log.Info().Str("assert", assert).Str("expect", expected).Msg("validate success")
+	log.Info().Str("check", check).Str("assert", assert).
+		Str("expect", expected).Msg("validate success")
 	return nil
 }
 

uixt/harmony_driver_hdc.go

@@ -288,7 +288,27 @@ func (hd *HDCDriver) PushImage(localPath string) error {
 	return nil
 }
 
+func (hd *HDCDriver) PullImages(localDir string) error {
+	log.Warn().Msg("PullImages not implemented in HDCDriver")
+	return nil
+}
+
 func (hd *HDCDriver) ClearImages() error {
 	log.Warn().Msg("ClearImages not implemented in HDCDriver")
 	return nil
 }
+
+func (hd *HDCDriver) PushFile(localPath string, remoteDir string) error {
+	log.Warn().Msg("PushFile not implemented in HDCDriver")
+	return nil
+}
+
+func (hd *HDCDriver) PullFiles(localDir string, remoteDirs ...string) error {
+	log.Warn().Msg("PullFiles not implemented in HDCDriver")
+	return nil
+}
+
+func (hd *HDCDriver) ClearFiles(paths ...string) error {
+	log.Warn().Msg("ClearFiles not implemented in HDCDriver")
+	return nil
+}

uixt/ios_driver_wda.go

@@ -989,6 +989,11 @@ func (wd *WDADriver) PushImage(localPath string) error {
 	return err
 }
 
+func (wd *WDADriver) PullImages(localDir string) error {
+	log.Warn().Msg("PullImages not implemented in WDADriver")
+	return nil
+}
+
 func (wd *WDADriver) ClearImages() error {
 	log.Info().Msg("WDADriver.ClearImages")
 	data := map[string]interface{}{}
@@ -997,6 +1002,21 @@ func (wd *WDADriver) ClearImages() error {
 	return err
 }
 
+func (wd *WDADriver) PushFile(localPath string, remoteDir string) error {
+	log.Warn().Msg("PushFile not implemented in WDADriver")
+	return nil
+}
+
+func (wd *WDADriver) PullFiles(localDir string, remoteDirs ...string) error {
+	log.Warn().Msg("PullFiles not implemented in WDADriver")
+	return nil
+}
+
+func (wd *WDADriver) ClearFiles(paths ...string) error {
+	log.Warn().Msg("ClearFiles not implemented in WDADriver")
+	return nil
+}
+
 type wdaResponse struct {
 	Status    int         `json:"status"`
 	SessionID string      `json:"sessionId"`