diff --git a/Makefile b/Makefile index f2f15434..1f1de82b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,13 @@ bump: ## bump hrp version, e.g. make bump version=4.0.0 .PHONY: build build: ## build hrp cli tool @echo "[info] build hrp cli tool" - @. scripts/build.sh $(tags) + go build -ldflags "\ + -s -w \ + -X 'github.com/httprunner/httprunner/v5/internal/version.GitCommit=$(shell git rev-parse HEAD)' \ + -X 'github.com/httprunner/httprunner/v5/internal/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)' \ + -X 'github.com/httprunner/httprunner/v5/internal/version.BuildTime=$(shell date "+%y%m%d%H%M")'" \ + -o output/hrp ./cmd/cli + ./output/hrp -v .PHONY: install-hooks install-hooks: ## install git hooks diff --git a/cmd/root.go b/cmd/root.go index aed93bd3..77bc8ae1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,7 +51,7 @@ Copyright © 2017-present debugtalk. Apache-2.0 License.`, PersistentPreRun: func(cmd *cobra.Command, args []string) { hrp.InitLogger(logLevel, logJSON) }, - Version: version.VERSION, + Version: version.GetVersionInfo(), TraverseChildren: true, // parses flags on all parents before executing child command SilenceUsage: true, // silence usage when an error occurs } diff --git a/internal/version/VERSION b/internal/version/VERSION index d9a653d2..b038c688 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503141055 +v5.0.0-beta-2503172040 diff --git a/internal/version/init.go b/internal/version/init.go deleted file mode 100644 index 2ae38ef8..00000000 --- a/internal/version/init.go +++ /dev/null @@ -1,13 +0,0 @@ -package version - -import ( - _ "embed" - "strings" -) - -//go:embed VERSION -var VERSION string - -func init() { - VERSION = strings.TrimSpace(VERSION) -} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 00000000..f936a5f4 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,31 @@ +package version + +import ( + "fmt" + "strings" + + _ "embed" +) + +//go:embed VERSION +var VERSION string + +func init() { + VERSION = strings.TrimSpace(VERSION) +} + +// 版本信息，在编译时通过 -ldflags 注入 +var ( + GitCommit = "unknown" + GitBranch = "unknown" + BuildTime = "unknown" +) + +func GetVersionInfo() string { + commitID := GitCommit + if len(commitID) > 8 { + commitID = commitID[:8] + } + return fmt.Sprintf("%s (branch=%s, commit=%s, build=%s)", + VERSION, GitBranch, commitID, BuildTime) +} diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100644 index ccb2a30e..00000000 --- a/scripts/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# build hrp cli binary for testing -# release will be triggered on github actions, see .github/workflows/release.yml - -# Usage: -# $ make build -# or -# $ bash scripts/build.sh - -set -e -set -x - -# prepare path -mkdir -p "output" -bin_path="output/hrp" - -# build -go build -ldflags '-s -w' -o "$bin_path" cmd/cli/main.go - -# check output and version -ls -lh "$bin_path" -chmod +x "$bin_path" -./"$bin_path" -v diff --git a/uixt/ai/cv.go b/uixt/ai/cv.go index fa314855..2fe70839 100644 --- a/uixt/ai/cv.go +++ b/uixt/ai/cv.go @@ -5,6 +5,7 @@ import ( "fmt" "image" "math" + "math/rand/v2" "regexp" "github.com/pkg/errors" @@ -86,19 +87,27 @@ func (t OCRText) Size() types.Size { } func (t OCRText) Center() PointF { - return getRectangleCenterPoint(t.Rect) -} - -func getRectangleCenterPoint(rect image.Rectangle) (point PointF) { + rect := t.Rect x, y := float64(rect.Min.X), float64(rect.Min.Y) width, height := float64(rect.Dx()), float64(rect.Dy()) - point = PointF{ + point := PointF{ X: x + width*0.5, Y: y + height*0.5, } return point } +func (t OCRText) RandomPoint() PointF { + rect := t.Rect + x, y := float64(rect.Min.X), float64(rect.Min.Y) + width, height := float64(rect.Dx()), float64(rect.Dy()) + point := PointF{ + X: x + width*rand.Float64(), + Y: y + height*rand.Float64(), + } + return point +} + type OCRTexts []OCRText func (t OCRTexts) texts() (texts []string) { @@ -240,6 +249,15 @@ func (box Box) Center() PointF { } } +func (box Box) RandomPoint() PointF { + width, height := float64(box.Width), float64(box.Height) + point := PointF{ + X: box.Point.X + width*rand.Float64(), + Y: box.Point.Y + height*rand.Float64(), + } + return point +} + type UIResults []UIResult func (u UIResults) FilterScope(scope option.AbsScope) (results UIResults) { diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index e812852d..ed5c7ab6 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -173,6 +173,7 @@ func (ad *ADBDriver) WindowSize() (size types.Size, err error) { // Back simulates a short press on the BACK button. func (ad *ADBDriver) Back() (err error) { + log.Info().Msg("ADBDriver.Back") // adb shell input keyevent 4 _, err = ad.runShellCommand("input", "keyevent", fmt.Sprintf("%d", KCBack)) if err != nil { @@ -200,10 +201,12 @@ func (ad *ADBDriver) Orientation() (orientation types.Orientation, err error) { } func (ad *ADBDriver) Home() (err error) { + log.Info().Msg("ADBDriver.Home") return ad.PressKeyCode(KCHome, KMEmpty) } func (ad *ADBDriver) Unlock() (err error) { + log.Info().Msg("ADBDriver.Unlock") // Notice: brighten should be executed before unlock // brighten android device screen if err := ad.PressKeyCode(KCWakeup, KMEmpty); err != nil { @@ -219,6 +222,7 @@ func (ad *ADBDriver) Unlock() (err error) { } func (ad *ADBDriver) Backspace(count int, opts ...option.ActionOption) (err error) { + log.Info().Int("count", count).Msg("ADBDriver.Backspace") if count == 0 { return nil } @@ -263,6 +267,7 @@ func (ad *ADBDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta) (err error } func (ad *ADBDriver) AppLaunch(packageName string) (err error) { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppLaunch") // 不指定 Activity 名称启动（启动主 Activity） // adb shell monkey -p -c android.intent.category.LAUNCHER 1 sOutput, err := ad.runShellCommand( @@ -280,6 +285,7 @@ func (ad *ADBDriver) AppLaunch(packageName string) (err error) { } func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err error) { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppTerminate") // 强制停止应用，停止 相关的进程 // adb shell am force-stop _, err = ad.runShellCommand("am", "force-stop", packageName) @@ -291,6 +297,7 @@ func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err erro } func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapXY") absX, absY, err := convertToAbsolutePoint(ad, x, y) if err != nil { return err @@ -299,8 +306,9 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY") actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -314,13 +322,14 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap") var err error x, y, err = convertToAbsolutePoint(ad, x, y) if err != nil { return err } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -340,8 +349,9 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error } func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := 1000.0 if actionOptions.Duration > 0 { duration = actionOptions.Duration * 1000 @@ -360,11 +370,14 @@ func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er } func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag") actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200.0 if actionOptions.Duration > 0 { @@ -388,11 +401,15 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO } func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe") var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) if err != nil { return err } + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) // adb shell input swipe fromX fromY toX toY _, err = ad.runShellCommand( @@ -416,6 +433,7 @@ func (ad *ADBDriver) ForceTouchFloat(x, y, pressure float64, second ...float64) } func (ad *ADBDriver) Input(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("ADBDriver.Input") err := ad.SendUnicodeKeys(text, opts...) if err == nil { return nil @@ -433,6 +451,7 @@ func (ad *ADBDriver) input(text string, _ ...option.ActionOption) error { } func (ad *ADBDriver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("ADBDriver.SendUnicodeKeys") // If the Unicode IME is not installed, fall back to the old interface. // There might be differences in the tracking schemes across different phones, and it is pending further verification. // In release version: without the Unicode IME installed, the test cannot execute. @@ -520,6 +539,7 @@ func (ad *ADBDriver) SendKeysByAdbKeyBoard(text string) (err error) { } func (ad *ADBDriver) AppClear(packageName string) error { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppClear") if _, err := ad.runShellCommand("pm", "clear", packageName); err != nil { log.Error().Str("packageName", packageName).Err(err).Msg("failed to clear package cache") return err @@ -549,6 +569,7 @@ func (ad *ADBDriver) ScreenShot(opts ...option.ActionOption) (raw *bytes.Buffer, } func (ad *ADBDriver) TapByHierarchy(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("ADBDriver.TapByHierarchy") sourceTree, err := ad.sourceTree() if err != nil { return err @@ -781,6 +802,7 @@ func (ad *ADBDriver) GetIme() (ime string, err error) { } func (ad *ADBDriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) { + log.Info().Msg("ADBDriver.ScreenRecord") options := option.NewActionOptions(opts...) var filePath string @@ -921,6 +943,7 @@ func (ad *ADBDriver) OpenUrl(url string) (err error) { } func (ad *ADBDriver) PushImage(localPath string) error { + log.Info().Str("localPath", localPath).Msg("ADBDriver.PushImage") remotePath := path.Join("/sdcard/DCIM/Camera/", path.Base(localPath)) if err := ad.Device.PushFile(localPath, remotePath); err != nil { return err @@ -933,6 +956,7 @@ func (ad *ADBDriver) PushImage(localPath string) error { } func (ad *ADBDriver) ClearImages() error { + log.Info().Msg("ADBDriver.ClearImages") _, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*") return nil } diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 938afc13..88416934 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -215,6 +215,7 @@ func (ud *UIA2Driver) WindowSize() (size types.Size, err error) { // Back simulates a short press on the BACK button. func (ud *UIA2Driver) Back() (err error) { + log.Info().Msg("UIA2Driver.Back") // register(postHandler, new PressBack("/wd/hub/session/:sessionId/back")) urlStr := fmt.Sprintf("/session/%s/back", ud.Session.ID) _, err = ud.Session.POST(nil, urlStr) @@ -253,13 +254,14 @@ func (ud *UIA2Driver) Orientation() (orientation types.Orientation, err error) { } func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap") var err error x, y, err = convertToAbsolutePoint(ud, x, y) if err != nil { return err } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) data := map[string]interface{}{ "actions": []interface{}{ @@ -284,6 +286,7 @@ func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error } func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapXY") // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) absX, absY, err := convertToAbsolutePoint(ud, x, y) if err != nil { @@ -293,9 +296,10 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY") // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := 100.0 if actionOptions.PressDuration > 0 { @@ -324,8 +328,9 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error } func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := actionOptions.Duration if duration == 0 { duration = 1.0 @@ -348,11 +353,16 @@ func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (e // Each step execution is throttled to 5 milliseconds per step, so for a 100 // steps, the swipe will take around 0.5 seconds to complete. func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag") var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY) if err != nil { return err } + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -374,12 +384,15 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action // `steps` is the number of move steps sent to the system func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { // register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform")) + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe") var err error actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY) if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200.0 if actionOptions.PressDuration > 0 { @@ -408,6 +421,8 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio } func (ud *UIA2Driver) SetPasteboard(contentType types.PasteboardType, content string) (err error) { + log.Info().Str("contentType", string(contentType)). + Str("content", content).Msg("UIA2Driver.SetPasteboard") lbl := content const defaultLabelLen = 10 @@ -454,6 +469,7 @@ func (ud *UIA2Driver) GetPasteboard(contentType types.PasteboardType) (raw *byte // SendKeys Android input does not support setting frequency. func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.Input") // register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys")) // https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85 err = ud.SendUnicodeKeys(text, opts...) @@ -471,6 +487,7 @@ func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error } func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.SendUnicodeKeys") // If the Unicode IME is not installed, fall back to the old interface. // There might be differences in the tracking schemes across different phones, and it is pending further verification. // In release version: without the Unicode IME installed, the test cannot execute. @@ -501,6 +518,7 @@ func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) } func (ud *UIA2Driver) SendActionKey(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.SendActionKey") var actions []interface{} for i, c := range text { actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)}, diff --git a/uixt/driver_ext_screenshot.go b/uixt/driver_ext_screenshot.go index 2ace7882..674b4a5f 100644 --- a/uixt/driver_ext_screenshot.go +++ b/uixt/driver_ext_screenshot.go @@ -145,7 +145,7 @@ func (dExt *XTDriver) GetScreenTexts(opts ...option.ActionOption) (ocrTexts ai.O return screenResult.Texts, nil } -func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (point ai.PointF, err error) { +func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (textRect ai.OCRText, err error) { options := option.NewActionOptions(opts...) if options.ScreenShotFileName == "" { opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("find_screen_text_%s", text))) @@ -155,19 +155,18 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) ( return } - result, err := ocrTexts.FindText(text, opts...) + textRect, err = ocrTexts.FindText(text, opts...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return } - point = result.Center() log.Info().Str("text", text). - Interface("point", point).Msgf("FindScreenText success") - return + Interface("textRect", textRect).Msgf("FindScreenText success") + return textRect, nil } -func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) { +func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (uiResult ai.UIResult, err error) { options := option.NewActionOptions(opts...) if options.ScreenShotFileName == "" { opts = append(opts, option.WithScreenShotFileName( @@ -183,11 +182,10 @@ func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF if err != nil { return } - uiResult, err := uiResults.GetUIResult(opts...) - point = uiResult.Center() + uiResult, err = uiResults.GetUIResult(opts...) log.Info().Interface("text", options.ScreenShotWithUITypes). - Interface("point", point).Msg("FindUIResult success") + Interface("uiResult", uiResult).Msg("FindUIResult success") return } diff --git a/uixt/driver_ext_swipe.go b/uixt/driver_ext_swipe.go index 9cb43574..78d1a532 100644 --- a/uixt/driver_ext_swipe.go +++ b/uixt/driver_ext_swipe.go @@ -156,7 +156,7 @@ func (dExt *XTDriver) SwipeToTapApp(appName string, opts ...option.ActionOption) actionOptions := option.NewActionOptions(opts...) // tap app icon above the text - if len(actionOptions.Offset) == 0 { + if len(actionOptions.TapOffset) == 0 { opts = append(opts, option.WithTapOffset(0, -25)) } // set default swipe interval to 1 second diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index b4e8f151..05dea676 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -3,7 +3,9 @@ package uixt import ( "fmt" + "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" + "github.com/rs/zerolog/log" ) func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { @@ -12,7 +14,7 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("tap_by_ocr_%s", text))) } - point, err := dExt.FindScreenText(text, opts...) + textRect, err := dExt.FindScreenText(text, opts...) if err != nil { if actionOptions.IgnoreNotFoundError { return nil @@ -20,19 +22,37 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { return err } + var point ai.PointF + if actionOptions.TapRandomRect { + point = textRect.RandomPoint() + } else { + point = textRect.Center() + } + log.Info().Str("text", text).Interface("rawTextRect", textRect). + Interface("tapPoint", point).Msg("TapByOCR success") + return dExt.TapAbsXY(point.X, point.Y, opts...) } func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error { - options := option.NewActionOptions(opts...) + actionOptions := option.NewActionOptions(opts...) - point, err := dExt.FindUIResult(opts...) + uiResult, err := dExt.FindUIResult(opts...) if err != nil { - if options.IgnoreNotFoundError { + if actionOptions.IgnoreNotFoundError { return nil } return err } + var point ai.PointF + if actionOptions.TapRandomRect { + point = uiResult.RandomPoint() + } else { + point = uiResult.Center() + } + log.Info().Interface("rawUIResult", uiResult). + Interface("tapPoint", point).Msg("TapByCV success") + return dExt.TapAbsXY(point.X, point.Y, opts...) } diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index 9afdcd15..243d1493 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -47,8 +47,8 @@ func TestDriverExt(t *testing.T) { driverExt.TapByOCR("推荐") texts, _ := driverExt.GetScreenTexts() t.Log(texts) - point, _ := driverExt.FindScreenText("hello") - t.Log(point) + textRect, _ := driverExt.FindScreenText("hello") + t.Log(textRect) err := driverExt.TapByCV( option.WithScreenShotUITypes("deepseek_send"), @@ -98,13 +98,14 @@ func TestDriverExt_FindScreenText(t *testing.T) { func TestDriverExt_Seek(t *testing.T) { driver := setupDriverExt(t) - point, err := driver.FindScreenText("首页") + textRect, err := driver.FindScreenText("首页") assert.Nil(t, err) size, err := driver.WindowSize() assert.Nil(t, err) width := size.Width + point := textRect.Center() y := point.Y - 40 for i := 0; i < 5; i++ { err := driver.Swipe(0.5, 0.8, 0.5, 0.2) @@ -168,3 +169,39 @@ func TestDriverExt_ClosePopupsHandler(t *testing.T) { err := driver.ClosePopupsHandler() assert.Nil(t, err) } + +func TestDriverExt_Action_Offset(t *testing.T) { + driver := setupADBDriverExt(t) + + // tap point with constant offset + err := driver.TapXY(0.5, 0.5, option.WithTapOffset(-10, 10)) + assert.Nil(t, err) + + // tap point with random offset + err = driver.TapXY(0.5, 0.5, option.WithOffsetRandomRange(-10, 10)) + assert.Nil(t, err) + + // swipe direction with constant offset + err = driver.Swipe(0.5, 0.5, 0.5, 0.9, + option.WithSwipeOffset(-50, 50, -50, 50)) + assert.Nil(t, err) + + // swipe direction with random offset + err = driver.Swipe(0.5, 0.5, 0.5, 0.9, + option.WithOffsetRandomRange(-50, 50)) + assert.Nil(t, err) + + // drag direction with random offset + err = driver.Drag(0.5, 0.5, 0.5, 0.9, + option.WithOffsetRandomRange(-50, 50)) + assert.Nil(t, err) + + // tap random point in ocr text rect + err = driver.TapByOCR("首页", option.WithTapRandomRect(true)) + assert.Nil(t, err) + + err = driver.TapByCV( + option.WithScreenShotUITypes("deepseek_send"), + option.WithTapRandomRect(true)) + assert.Nil(t, err) +} diff --git a/uixt/harmony_driver_hdc.go b/uixt/harmony_driver_hdc.go index fb248791..1541245f 100644 --- a/uixt/harmony_driver_hdc.go +++ b/uixt/harmony_driver_hdc.go @@ -97,6 +97,7 @@ const ( ) func (hd *HDCDriver) Unlock() (err error) { + log.Info().Msg("HDCDriver.Unlock") // Todo 检查是否锁屏 hdc shell hidumper -s RenderService -a screen screenInfo, err := hd.Device.RunShellCommand("hidumper", "-s", "RenderService", "-a", "screen") if err != nil { @@ -124,6 +125,7 @@ func (hd *HDCDriver) AppLaunch(packageName string) error { } func (hd *HDCDriver) AppTerminate(packageName string) (bool, error) { + log.Info().Str("packageName", packageName).Msg("HDCDriver.AppTerminate") _, err := hd.Device.RunShellCommand("aa", "force-stop", packageName) if err != nil { log.Error().Err(err).Msg("failed to terminal app") @@ -142,6 +144,7 @@ func (hd *HDCDriver) Orientation() (orientation types.Orientation, err error) { } func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapXY") absX, absY, err := convertToAbsolutePoint(hd, x, y) if err != nil { return err @@ -150,8 +153,9 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY") actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) if actionOptions.Identifier != "" { startTime := int(time.Now().UnixMilli()) @@ -175,12 +179,15 @@ func (hd *HDCDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO // Swipe works like Drag, but `pressForDuration` value is 0 func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe") var err error actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(hd, fromX, fromY, toX, toY) if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200 if actionOptions.PressDuration > 0 { @@ -208,6 +215,7 @@ func (hd *HDCDriver) AppClear(packageName string) error { } func (hd *HDCDriver) Back() error { + log.Info().Msg("HDCDriver.Back") return hd.uiDriver.PressBack() } diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 8a97a19b..0c56aa66 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -108,7 +108,7 @@ func (wd *WDADriver) initMjpegClient() error { } mjpegHTTPConn, err := net.Dial( "tcp", - fmt.Sprintf("%s:%d", host, localMjpegPort), + net.JoinHostPort(host, fmt.Sprintf("%d", localMjpegPort)), ) if err != nil { return errors.Wrap(code.DeviceHTTPDriverError, err.Error()) @@ -492,6 +492,7 @@ func (wd *WDADriver) AlertSendKeys(text string) (err error) { } func (wd *WDADriver) AppLaunch(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunch") // [[FBRoute POST:@"/wda/apps/launch"] respondWithTarget:self action:@selector(handleSessionAppLaunch:)] data := make(map[string]interface{}) data["bundleId"] = bundleId @@ -508,6 +509,7 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) { } func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunchUnattached") // [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)] data := map[string]interface{}{"bundleId": bundleId} _, err = wd.Session.POST(data, "/wda/apps/launchUnattached") @@ -519,6 +521,7 @@ func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) { } func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppTerminate") // [[FBRoute POST:@"/wda/apps/terminate"] respondWithTarget:self action:@selector(handleSessionAppTerminate:)] data := map[string]interface{}{"bundleId": bundleId} var rawResp DriverRawResponse @@ -533,6 +536,7 @@ func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error) } func (wd *WDADriver) AppActivate(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppActivate") // [[FBRoute POST:@"/wda/apps/activate"] respondWithTarget:self action:@selector(handleSessionAppActivate:)] data := map[string]interface{}{"bundleId": bundleId} urlStr := fmt.Sprintf("/session/%s/wda/apps/activate", wd.Session.ID) @@ -541,6 +545,7 @@ func (wd *WDADriver) AppActivate(bundleId string) (err error) { } func (wd *WDADriver) AppDeactivate(second float64) (err error) { + log.Info().Float64("second", second).Msg("WDADriver.AppDeactivate") // [[FBRoute POST:@"/wda/deactivateApp"] respondWithTarget:self action:@selector(handleDeactivateAppCommand:)] if second < 3 { second = 3.0 @@ -576,6 +581,7 @@ func (wd *WDADriver) ForegroundInfo() (appInfo types.AppInfo, err error) { } func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapXY") // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] absX, absY, err := convertToAbsolutePoint(wd, x, y) if err != nil { @@ -585,9 +591,10 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY") // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) data := map[string]interface{}{ "x": wd.toScale(x), "y": wd.toScale(y), @@ -600,6 +607,7 @@ func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap") // [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)] var err error x, y, err = convertToAbsolutePoint(wd, x, y) @@ -608,7 +616,7 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) x = wd.toScale(x) y = wd.toScale(y) data := map[string]interface{}{ @@ -622,8 +630,9 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error // FIXME: hold not work func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) if actionOptions.Duration == 0 { opts = append(opts, option.WithPressDuration(1)) } @@ -631,6 +640,8 @@ func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er } func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag") // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(wd, fromX, fromY, toX, toY) @@ -641,6 +652,9 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO fromY = wd.toScale(fromY) toX = wd.toScale(toX) toY = wd.toScale(toY) + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + data := map[string]interface{}{ "fromX": math.Round(fromX*10) / 10, "fromY": math.Round(fromY*10) / 10, @@ -689,6 +703,7 @@ func (wd *WDADriver) SetIme(ime string) error { } func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("WDADriver.Input") // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] data := map[string]interface{}{"value": strings.Split(text, "")} option.MergeOptions(data, opts...) @@ -698,6 +713,7 @@ func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) } func (wd *WDADriver) Backspace(count int, opts ...option.ActionOption) (err error) { + log.Info().Int("count", count).Msg("WDADriver.Backspace") if count == 0 { return nil } @@ -713,6 +729,7 @@ func (wd *WDADriver) AppClear(packageName string) error { // Back simulates a short press on the BACK button. func (wd *WDADriver) Back() (err error) { + log.Info().Msg("WDADriver.Back") return wd.Swipe(0, 0.5, 0.6, 0.5) } @@ -876,6 +893,7 @@ func (wd *WDADriver) triggerWDALog(data map[string]interface{}) (rawResp []byte, } func (wd *WDADriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) { + log.Info().Msg("WDADriver.ScreenRecord") timestamp := time.Now().Format("20060102_150405") + fmt.Sprintf("_%03d", time.Now().UnixNano()/1e6%1000) fileName := filepath.Join(config.GetConfig().ScreenShotsPath, fmt.Sprintf("%s.mp4", timestamp)) @@ -951,6 +969,7 @@ func (wd *WDADriver) StartCaptureLog(identifier ...string) error { } func (wd *WDADriver) PushImage(localPath string) error { + log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage") localFile, err := os.Open(localPath) if err != nil { return err @@ -971,6 +990,7 @@ func (wd *WDADriver) PushImage(localPath string) error { } func (wd *WDADriver) ClearImages() error { + log.Info().Msg("WDADriver.ClearImages") data := map[string]interface{}{} _, err := wd.Session.POST(data, "/gtf/albums/clear") diff --git a/uixt/option/action.go b/uixt/option/action.go index 594a6cd4..72f04c18 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -95,13 +95,18 @@ func (o *ActionOptions) Options() []ActionOption { options = append(options, WithScope( o.Scope[0], o.Scope[1], o.Scope[2], o.Scope[3])) } - if len(o.Offset) == 2 { + if len(o.TapOffset) == 2 { // for tap [x,y] offset - options = append(options, WithTapOffset(o.Offset[0], o.Offset[1])) - } else if len(o.Offset) == 4 { + options = append(options, WithTapOffset(o.TapOffset[0], o.TapOffset[1])) + } + if o.TapRandomRect { + // tap random point in OCR/CV rectangle + options = append(options, WithTapRandomRect(true)) + } + if len(o.SwipeOffset) == 4 { // for swipe [fromX, fromY, toX, toY] offset options = append(options, WithSwipeOffset( - o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3])) + o.SwipeOffset[0], o.SwipeOffset[1], o.SwipeOffset[2], o.SwipeOffset[3])) } if len(o.OffsetRandomRange) == 2 { options = append(options, WithOffsetRandomRange( @@ -131,17 +136,32 @@ func (o *ActionOptions) Options() []ActionOption { return options } -func (o *ActionOptions) ApplyOffset(absX, absY float64) (float64, float64) { - if len(o.Offset) == 2 { - absX += float64(o.Offset[0]) - absY += float64(o.Offset[1]) +func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) { + if len(o.TapOffset) == 2 { + absX += float64(o.TapOffset[0]) + absY += float64(o.TapOffset[1]) } - absX += o.GenerateRandomOffset() - absY += o.GenerateRandomOffset() + absX += o.generateRandomOffset() + absY += o.generateRandomOffset() return absX, absY } -func (o *ActionOptions) GenerateRandomOffset() float64 { +func (o *ActionOptions) ApplySwipeOffset(absFromX, absFromY, absToX, absToY float64) ( + float64, float64, float64, float64) { + if len(o.SwipeOffset) == 4 { + absFromX += float64(o.SwipeOffset[0]) + absFromY += float64(o.SwipeOffset[1]) + absToX += float64(o.SwipeOffset[2]) + absToY += float64(o.SwipeOffset[3]) + } + absFromX += o.generateRandomOffset() + absFromY += o.generateRandomOffset() + absToX += o.generateRandomOffset() + absToY += o.generateRandomOffset() + return absFromX, absFromY, absToX, absToY +} + +func (o *ActionOptions) generateRandomOffset() float64 { if len(o.OffsetRandomRange) != 2 { // invalid offset random range, should be [min, max] return 0 @@ -276,7 +296,7 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { // swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY] func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption { return func(o *ActionOptions) { - o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY} + o.SwipeOffset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY} } } diff --git a/uixt/option/screen.go b/uixt/option/screen.go index 160dd4c9..feb3772f 100644 --- a/uixt/option/screen.go +++ b/uixt/option/screen.go @@ -216,7 +216,9 @@ type ScreenFilterOptions struct { AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + TapOffset []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"` // tap with absolute point offset + TapRandomRect bool `json:"tap_random_rect,omitempty" yaml:"tap_random_rect,omitempty"` // tap random point in text/image rectangle + SwipeOffset []int `json:"swipe_offset,omitempty" yaml:"swipe_offset,omitempty"` // swipe with direction offset OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element MatchOne bool `json:"match_one,omitempty" yaml:"match_one,omitempty"` @@ -242,7 +244,15 @@ func WithAbsScope(x1, y1, x2, y2 int) ActionOption { // tap [x, y] with offset [offsetX, offsetY] func WithTapOffset(offsetX, offsetY int) ActionOption { return func(o *ActionOptions) { - o.Offset = []int{offsetX, offsetY} + o.TapOffset = []int{offsetX, offsetY} + } +} + +// WithTapRandomRect is used with TapByOCR and TapByCV +// when set true, tap random point in text/image rectangle +func WithTapRandomRect(tapRandom bool) ActionOption { + return func(o *ActionOptions) { + o.TapRandomRect = tapRandom } }