Merge branch 'risk' into 'v5'

行为风控对抗：针对 UI 操作引入随机能力

See merge request iesqa/httprunner!75

Makefile

@@ -16,7 +16,13 @@ bump: ## bump hrp version, e.g. make bump version=4.0.0
 .PHONY: build
 build: ## build hrp cli tool
 	@echo "[info] build hrp cli tool"
-	@. scripts/build.sh $(tags)
+	go build -ldflags "\
+		-s -w \
+		-X 'github.com/httprunner/httprunner/v5/internal/version.GitCommit=$(shell git rev-parse HEAD)' \
+		-X 'github.com/httprunner/httprunner/v5/internal/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)' \
+		-X 'github.com/httprunner/httprunner/v5/internal/version.BuildTime=$(shell date "+%y%m%d%H%M")'" \
+		-o output/hrp ./cmd/cli
+	./output/hrp -v
 
 .PHONY: install-hooks
 install-hooks: ## install git hooks

cmd/root.go

@@ -51,7 +51,7 @@ Copyright © 2017-present debugtalk. Apache-2.0 License.`,
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		hrp.InitLogger(logLevel, logJSON)
 	},
-	Version:          version.VERSION,
+	Version:          version.GetVersionInfo(),
 	TraverseChildren: true, // parses flags on all parents before executing child command
 	SilenceUsage:     true, // silence usage when an error occurs
 }

internal/version/VERSION

@@ -1 +1 @@
-v5.0.0-beta-2503141055
+v5.0.0-beta-2503172040

internal/version/init.go

@@ -1,13 +0,0 @@
-package version
-
-import (
-	_ "embed"
-	"strings"
-)
-
-//go:embed VERSION
-var VERSION string
-
-func init() {
-	VERSION = strings.TrimSpace(VERSION)
-}

internal/version/version.go

@@ -0,0 +1,31 @@
+package version
+
+import (
+	"fmt"
+	"strings"
+
+	_ "embed"
+)
+
+//go:embed VERSION
+var VERSION string
+
+func init() {
+	VERSION = strings.TrimSpace(VERSION)
+}
+
+// 版本信息，在编译时通过 -ldflags 注入
+var (
+	GitCommit = "unknown"
+	GitBranch = "unknown"
+	BuildTime = "unknown"
+)
+
+func GetVersionInfo() string {
+	commitID := GitCommit
+	if len(commitID) > 8 {
+		commitID = commitID[:8]
+	}
+	return fmt.Sprintf("%s (branch=%s, commit=%s, build=%s)",
+		VERSION, GitBranch, commitID, BuildTime)
+}

scripts/build.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# build hrp cli binary for testing
-# release will be triggered on github actions, see .github/workflows/release.yml
-
-# Usage:
-# $ make build
-# or
-# $ bash scripts/build.sh
-
-set -e
-set -x
-
-# prepare path
-mkdir -p "output"
-bin_path="output/hrp"
-
-# build
-go build -ldflags '-s -w' -o "$bin_path" cmd/cli/main.go
-
-# check output and version
-ls -lh "$bin_path"
-chmod +x "$bin_path"
-./"$bin_path" -v

uixt/ai/cv.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"image"
 	"math"
+	"math/rand/v2"
 	"regexp"
 
 	"github.com/pkg/errors"
@@ -86,19 +87,27 @@ func (t OCRText) Size() types.Size {
 }
 
 func (t OCRText) Center() PointF {
-	return getRectangleCenterPoint(t.Rect)
+	rect := t.Rect
-}
-
-func getRectangleCenterPoint(rect image.Rectangle) (point PointF) {
 	x, y := float64(rect.Min.X), float64(rect.Min.Y)
 	width, height := float64(rect.Dx()), float64(rect.Dy())
-	point = PointF{
+	point := PointF{
 		X: x + width*0.5,
 		Y: y + height*0.5,
 	}
 	return point
 }
 
+func (t OCRText) RandomPoint() PointF {
+	rect := t.Rect
+	x, y := float64(rect.Min.X), float64(rect.Min.Y)
+	width, height := float64(rect.Dx()), float64(rect.Dy())
+	point := PointF{
+		X: x + width*rand.Float64(),
+		Y: y + height*rand.Float64(),
+	}
+	return point
+}
+
 type OCRTexts []OCRText
 
 func (t OCRTexts) texts() (texts []string) {
@@ -240,6 +249,15 @@ func (box Box) Center() PointF {
 	}
 }
 
+func (box Box) RandomPoint() PointF {
+	width, height := float64(box.Width), float64(box.Height)
+	point := PointF{
+		X: box.Point.X + width*rand.Float64(),
+		Y: box.Point.Y + height*rand.Float64(),
+	}
+	return point
+}
+
 type UIResults []UIResult
 
 func (u UIResults) FilterScope(scope option.AbsScope) (results UIResults) {

uixt/android_driver_adb.go

@@ -173,6 +173,7 @@ func (ad *ADBDriver) WindowSize() (size types.Size, err error) {
 
 // Back simulates a short press on the BACK button.
 func (ad *ADBDriver) Back() (err error) {
+	log.Info().Msg("ADBDriver.Back")
 	// adb shell input keyevent 4
 	_, err = ad.runShellCommand("input", "keyevent", fmt.Sprintf("%d", KCBack))
 	if err != nil {
@@ -200,10 +201,12 @@ func (ad *ADBDriver) Orientation() (orientation types.Orientation, err error) {
 }
 
 func (ad *ADBDriver) Home() (err error) {
+	log.Info().Msg("ADBDriver.Home")
 	return ad.PressKeyCode(KCHome, KMEmpty)
 }
 
 func (ad *ADBDriver) Unlock() (err error) {
+	log.Info().Msg("ADBDriver.Unlock")
 	// Notice: brighten should be executed before unlock
 	// brighten android device screen
 	if err := ad.PressKeyCode(KCWakeup, KMEmpty); err != nil {
@@ -219,6 +222,7 @@ func (ad *ADBDriver) Unlock() (err error) {
 }
 
 func (ad *ADBDriver) Backspace(count int, opts ...option.ActionOption) (err error) {
+	log.Info().Int("count", count).Msg("ADBDriver.Backspace")
 	if count == 0 {
 		return nil
 	}
@@ -263,6 +267,7 @@ func (ad *ADBDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta) (err error
 }
 
 func (ad *ADBDriver) AppLaunch(packageName string) (err error) {
+	log.Info().Str("packageName", packageName).Msg("ADBDriver.AppLaunch")
 	// 不指定 Activity 名称启动（启动主 Activity）
 	// adb shell monkey -p <packagename> -c android.intent.category.LAUNCHER 1
 	sOutput, err := ad.runShellCommand(
@@ -280,6 +285,7 @@ func (ad *ADBDriver) AppLaunch(packageName string) (err error) {
 }
 
 func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err error) {
+	log.Info().Str("packageName", packageName).Msg("ADBDriver.AppTerminate")
 	// 强制停止应用，停止 <packagename> 相关的进程
 	// adb shell am force-stop <packagename>
 	_, err = ad.runShellCommand("am", "force-stop", packageName)
@@ -291,6 +297,7 @@ func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err erro
 }
 
 func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapXY")
 	absX, absY, err := convertToAbsolutePoint(ad, x, y)
 	if err != nil {
 		return err
@@ -299,8 +306,9 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY")
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 
 	// adb shell input tap x y
 	xStr := fmt.Sprintf("%.1f", x)
@@ -314,13 +322,14 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap")
 	var err error
 	x, y, err = convertToAbsolutePoint(ad, x, y)
 	if err != nil {
 		return err
 	}
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 
 	// adb shell input tap x y
 	xStr := fmt.Sprintf("%.1f", x)
@@ -340,8 +349,9 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error
 }
 
 func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) {
+	log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TouchAndHold")
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 	duration := 1000.0
 	if actionOptions.Duration > 0 {
 		duration = actionOptions.Duration * 1000
@@ -360,11 +370,14 @@ func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er
 }
 
 func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) {
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag")
 	actionOptions := option.NewActionOptions(opts...)
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY)
 	if err != nil {
 		return err
 	}
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
 
 	duration := 200.0
 	if actionOptions.Duration > 0 {
@@ -388,11 +401,15 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO
 }
 
 func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe")
 	var err error
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY)
 	if err != nil {
 		return err
 	}
+	actionOptions := option.NewActionOptions(opts...)
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
 
 	// adb shell input swipe fromX fromY toX toY
 	_, err = ad.runShellCommand(
@@ -416,6 +433,7 @@ func (ad *ADBDriver) ForceTouchFloat(x, y, pressure float64, second ...float64)
 }
 
 func (ad *ADBDriver) Input(text string, opts ...option.ActionOption) error {
+	log.Info().Str("text", text).Msg("ADBDriver.Input")
 	err := ad.SendUnicodeKeys(text, opts...)
 	if err == nil {
 		return nil
@@ -433,6 +451,7 @@ func (ad *ADBDriver) input(text string, _ ...option.ActionOption) error {
 }
 
 func (ad *ADBDriver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) {
+	log.Info().Str("text", text).Msg("ADBDriver.SendUnicodeKeys")
 	// If the Unicode IME is not installed, fall back to the old interface.
 	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
 	// In release version: without the Unicode IME installed, the test cannot execute.
@@ -520,6 +539,7 @@ func (ad *ADBDriver) SendKeysByAdbKeyBoard(text string) (err error) {
 }
 
 func (ad *ADBDriver) AppClear(packageName string) error {
+	log.Info().Str("packageName", packageName).Msg("ADBDriver.AppClear")
 	if _, err := ad.runShellCommand("pm", "clear", packageName); err != nil {
 		log.Error().Str("packageName", packageName).Err(err).Msg("failed to clear package cache")
 		return err
@@ -549,6 +569,7 @@ func (ad *ADBDriver) ScreenShot(opts ...option.ActionOption) (raw *bytes.Buffer,
 }
 
 func (ad *ADBDriver) TapByHierarchy(text string, opts ...option.ActionOption) error {
+	log.Info().Str("text", text).Msg("ADBDriver.TapByHierarchy")
 	sourceTree, err := ad.sourceTree()
 	if err != nil {
 		return err
@@ -781,6 +802,7 @@ func (ad *ADBDriver) GetIme() (ime string, err error) {
 }
 
 func (ad *ADBDriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) {
+	log.Info().Msg("ADBDriver.ScreenRecord")
 	options := option.NewActionOptions(opts...)
 
 	var filePath string
@@ -921,6 +943,7 @@ func (ad *ADBDriver) OpenUrl(url string) (err error) {
 }
 
 func (ad *ADBDriver) PushImage(localPath string) error {
+	log.Info().Str("localPath", localPath).Msg("ADBDriver.PushImage")
 	remotePath := path.Join("/sdcard/DCIM/Camera/", path.Base(localPath))
 	if err := ad.Device.PushFile(localPath, remotePath); err != nil {
 		return err
@@ -933,6 +956,7 @@ func (ad *ADBDriver) PushImage(localPath string) error {
 }
 
 func (ad *ADBDriver) ClearImages() error {
+	log.Info().Msg("ADBDriver.ClearImages")
 	_, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*")
 	return nil
 }

uixt/android_driver_uia2.go

@@ -215,6 +215,7 @@ func (ud *UIA2Driver) WindowSize() (size types.Size, err error) {
 
 // Back simulates a short press on the BACK button.
 func (ud *UIA2Driver) Back() (err error) {
+	log.Info().Msg("UIA2Driver.Back")
 	// register(postHandler, new PressBack("/wd/hub/session/:sessionId/back"))
 	urlStr := fmt.Sprintf("/session/%s/back", ud.Session.ID)
 	_, err = ud.Session.POST(nil, urlStr)
@@ -253,13 +254,14 @@ func (ud *UIA2Driver) Orientation() (orientation types.Orientation, err error) {
 }
 
 func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap")
 	var err error
 	x, y, err = convertToAbsolutePoint(ud, x, y)
 	if err != nil {
 		return err
 	}
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 
 	data := map[string]interface{}{
 		"actions": []interface{}{
@@ -284,6 +286,7 @@ func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error
 }
 
 func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapXY")
 	// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
 	absX, absY, err := convertToAbsolutePoint(ud, x, y)
 	if err != nil {
@@ -293,9 +296,10 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY")
 	// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 
 	duration := 100.0
 	if actionOptions.PressDuration > 0 {
@@ -324,8 +328,9 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error
 }
 
 func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) {
+	log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TouchAndHold")
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 	duration := actionOptions.Duration
 	if duration == 0 {
 		duration = 1.0
@@ -348,11 +353,16 @@ func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (e
 // Each step execution is throttled to 5 milliseconds per step, so for a 100
 // steps, the swipe will take around 0.5 seconds to complete.
 func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag")
 	var err error
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY)
 	if err != nil {
 		return err
 	}
+	actionOptions := option.NewActionOptions(opts...)
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
+
 	data := map[string]interface{}{
 		"startX": fromX,
 		"startY": fromY,
@@ -374,12 +384,15 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action
 //	`steps` is the number of move steps sent to the system
 func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
 	// register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform"))
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe")
 	var err error
 	actionOptions := option.NewActionOptions(opts...)
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY)
 	if err != nil {
 		return err
 	}
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
 
 	duration := 200.0
 	if actionOptions.PressDuration > 0 {
@@ -408,6 +421,8 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio
 }
 
 func (ud *UIA2Driver) SetPasteboard(contentType types.PasteboardType, content string) (err error) {
+	log.Info().Str("contentType", string(contentType)).
+		Str("content", content).Msg("UIA2Driver.SetPasteboard")
 	lbl := content
 
 	const defaultLabelLen = 10
@@ -454,6 +469,7 @@ func (ud *UIA2Driver) GetPasteboard(contentType types.PasteboardType) (raw *byte
 
 // SendKeys Android input does not support setting frequency.
 func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error) {
+	log.Info().Str("text", text).Msg("UIA2Driver.Input")
 	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
 	// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
 	err = ud.SendUnicodeKeys(text, opts...)
@@ -471,6 +487,7 @@ func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error
 }
 
 func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) {
+	log.Info().Str("text", text).Msg("UIA2Driver.SendUnicodeKeys")
 	// If the Unicode IME is not installed, fall back to the old interface.
 	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
 	// In release version: without the Unicode IME installed, the test cannot execute.
@@ -501,6 +518,7 @@ func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption)
 }
 
 func (ud *UIA2Driver) SendActionKey(text string, opts ...option.ActionOption) (err error) {
+	log.Info().Str("text", text).Msg("UIA2Driver.SendActionKey")
 	var actions []interface{}
 	for i, c := range text {
 		actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)},

uixt/driver_ext_screenshot.go

@@ -145,7 +145,7 @@ func (dExt *XTDriver) GetScreenTexts(opts ...option.ActionOption) (ocrTexts ai.O
 	return screenResult.Texts, nil
 }
 
-func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (point ai.PointF, err error) {
+func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (textRect ai.OCRText, err error) {
 	options := option.NewActionOptions(opts...)
 	if options.ScreenShotFileName == "" {
 		opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("find_screen_text_%s", text)))
@@ -155,19 +155,18 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (
 		return
 	}
 
-	result, err := ocrTexts.FindText(text, opts...)
+	textRect, err = ocrTexts.FindText(text, opts...)
 	if err != nil {
 		log.Warn().Msgf("FindText failed: %s", err.Error())
 		return
 	}
-	point = result.Center()
 
 	log.Info().Str("text", text).
-		Interface("point", point).Msgf("FindScreenText success")
+		Interface("textRect", textRect).Msgf("FindScreenText success")
-	return
+	return textRect, nil
 }
 
-func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) {
+func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (uiResult ai.UIResult, err error) {
 	options := option.NewActionOptions(opts...)
 	if options.ScreenShotFileName == "" {
 		opts = append(opts, option.WithScreenShotFileName(
@@ -183,11 +182,10 @@ func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF
 	if err != nil {
 		return
 	}
-	uiResult, err := uiResults.GetUIResult(opts...)
+	uiResult, err = uiResults.GetUIResult(opts...)
-	point = uiResult.Center()
 
 	log.Info().Interface("text", options.ScreenShotWithUITypes).
-		Interface("point", point).Msg("FindUIResult success")
+		Interface("uiResult", uiResult).Msg("FindUIResult success")
 	return
 }
 

uixt/driver_ext_swipe.go

@@ -156,7 +156,7 @@ func (dExt *XTDriver) SwipeToTapApp(appName string, opts ...option.ActionOption)
 
 	actionOptions := option.NewActionOptions(opts...)
 	// tap app icon above the text
-	if len(actionOptions.Offset) == 0 {
+	if len(actionOptions.TapOffset) == 0 {
 		opts = append(opts, option.WithTapOffset(0, -25))
 	}
 	// set default swipe interval to 1 second

uixt/driver_ext_tap.go

@@ -3,7 +3,9 @@ package uixt
 import (
 	"fmt"
 
+	"github.com/httprunner/httprunner/v5/uixt/ai"
 	"github.com/httprunner/httprunner/v5/uixt/option"
+	"github.com/rs/zerolog/log"
 )
 
 func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error {
@@ -12,7 +14,7 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error {
 		opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("tap_by_ocr_%s", text)))
 	}
 
-	point, err := dExt.FindScreenText(text, opts...)
+	textRect, err := dExt.FindScreenText(text, opts...)
 	if err != nil {
 		if actionOptions.IgnoreNotFoundError {
 			return nil
@@ -20,19 +22,37 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error {
 		return err
 	}
 
+	var point ai.PointF
+	if actionOptions.TapRandomRect {
+		point = textRect.RandomPoint()
+	} else {
+		point = textRect.Center()
+	}
+	log.Info().Str("text", text).Interface("rawTextRect", textRect).
+		Interface("tapPoint", point).Msg("TapByOCR success")
+
 	return dExt.TapAbsXY(point.X, point.Y, opts...)
 }
 
 func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error {
-	options := option.NewActionOptions(opts...)
+	actionOptions := option.NewActionOptions(opts...)
 
-	point, err := dExt.FindUIResult(opts...)
+	uiResult, err := dExt.FindUIResult(opts...)
 	if err != nil {
-		if options.IgnoreNotFoundError {
+		if actionOptions.IgnoreNotFoundError {
 			return nil
 		}
 		return err
 	}
 
+	var point ai.PointF
+	if actionOptions.TapRandomRect {
+		point = uiResult.RandomPoint()
+	} else {
+		point = uiResult.Center()
+	}
+	log.Info().Interface("rawUIResult", uiResult).
+		Interface("tapPoint", point).Msg("TapByCV success")
+
 	return dExt.TapAbsXY(point.X, point.Y, opts...)
 }

uixt/driver_ext_test.go

@@ -47,8 +47,8 @@ func TestDriverExt(t *testing.T) {
 	driverExt.TapByOCR("推荐")
 	texts, _ := driverExt.GetScreenTexts()
 	t.Log(texts)
-	point, _ := driverExt.FindScreenText("hello")
+	textRect, _ := driverExt.FindScreenText("hello")
-	t.Log(point)
+	t.Log(textRect)
 
 	err := driverExt.TapByCV(
 		option.WithScreenShotUITypes("deepseek_send"),
@@ -98,13 +98,14 @@ func TestDriverExt_FindScreenText(t *testing.T) {
 func TestDriverExt_Seek(t *testing.T) {
 	driver := setupDriverExt(t)
 
-	point, err := driver.FindScreenText("首页")
+	textRect, err := driver.FindScreenText("首页")
 	assert.Nil(t, err)
 
 	size, err := driver.WindowSize()
 	assert.Nil(t, err)
 	width := size.Width
 
+	point := textRect.Center()
 	y := point.Y - 40
 	for i := 0; i < 5; i++ {
 		err := driver.Swipe(0.5, 0.8, 0.5, 0.2)
@@ -168,3 +169,39 @@ func TestDriverExt_ClosePopupsHandler(t *testing.T) {
 	err := driver.ClosePopupsHandler()
 	assert.Nil(t, err)
 }
+
+func TestDriverExt_Action_Offset(t *testing.T) {
+	driver := setupADBDriverExt(t)
+
+	// tap point with constant offset
+	err := driver.TapXY(0.5, 0.5, option.WithTapOffset(-10, 10))
+	assert.Nil(t, err)
+
+	// tap point with random offset
+	err = driver.TapXY(0.5, 0.5, option.WithOffsetRandomRange(-10, 10))
+	assert.Nil(t, err)
+
+	// swipe direction with constant offset
+	err = driver.Swipe(0.5, 0.5, 0.5, 0.9,
+		option.WithSwipeOffset(-50, 50, -50, 50))
+	assert.Nil(t, err)
+
+	// swipe direction with random offset
+	err = driver.Swipe(0.5, 0.5, 0.5, 0.9,
+		option.WithOffsetRandomRange(-50, 50))
+	assert.Nil(t, err)
+
+	// drag direction with random offset
+	err = driver.Drag(0.5, 0.5, 0.5, 0.9,
+		option.WithOffsetRandomRange(-50, 50))
+	assert.Nil(t, err)
+
+	// tap random point in ocr text rect
+	err = driver.TapByOCR("首页", option.WithTapRandomRect(true))
+	assert.Nil(t, err)
+
+	err = driver.TapByCV(
+		option.WithScreenShotUITypes("deepseek_send"),
+		option.WithTapRandomRect(true))
+	assert.Nil(t, err)
+}

uixt/harmony_driver_hdc.go

@@ -97,6 +97,7 @@ const (
 )
 
 func (hd *HDCDriver) Unlock() (err error) {
+	log.Info().Msg("HDCDriver.Unlock")
 	// Todo 检查是否锁屏 hdc shell hidumper -s RenderService -a screen
 	screenInfo, err := hd.Device.RunShellCommand("hidumper", "-s", "RenderService", "-a", "screen")
 	if err != nil {
@@ -124,6 +125,7 @@ func (hd *HDCDriver) AppLaunch(packageName string) error {
 }
 
 func (hd *HDCDriver) AppTerminate(packageName string) (bool, error) {
+	log.Info().Str("packageName", packageName).Msg("HDCDriver.AppTerminate")
 	_, err := hd.Device.RunShellCommand("aa", "force-stop", packageName)
 	if err != nil {
 		log.Error().Err(err).Msg("failed to terminal app")
@@ -142,6 +144,7 @@ func (hd *HDCDriver) Orientation() (orientation types.Orientation, err error) {
 }
 
 func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapXY")
 	absX, absY, err := convertToAbsolutePoint(hd, x, y)
 	if err != nil {
 		return err
@@ -150,8 +153,9 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY")
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 
 	if actionOptions.Identifier != "" {
 		startTime := int(time.Now().UnixMilli())
@@ -175,12 +179,15 @@ func (hd *HDCDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO
 
 // Swipe works like Drag, but `pressForDuration` value is 0
 func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe")
 	var err error
 	actionOptions := option.NewActionOptions(opts...)
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(hd, fromX, fromY, toX, toY)
 	if err != nil {
 		return err
 	}
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
 
 	duration := 200
 	if actionOptions.PressDuration > 0 {
@@ -208,6 +215,7 @@ func (hd *HDCDriver) AppClear(packageName string) error {
 }
 
 func (hd *HDCDriver) Back() error {
+	log.Info().Msg("HDCDriver.Back")
 	return hd.uiDriver.PressBack()
 }
 

uixt/ios_driver_wda.go

@@ -108,7 +108,7 @@ func (wd *WDADriver) initMjpegClient() error {
 	}
 	mjpegHTTPConn, err := net.Dial(
 		"tcp",
-		fmt.Sprintf("%s:%d", host, localMjpegPort),
+		net.JoinHostPort(host, fmt.Sprintf("%d", localMjpegPort)),
 	)
 	if err != nil {
 		return errors.Wrap(code.DeviceHTTPDriverError, err.Error())
@@ -492,6 +492,7 @@ func (wd *WDADriver) AlertSendKeys(text string) (err error) {
 }
 
 func (wd *WDADriver) AppLaunch(bundleId string) (err error) {
+	log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunch")
 	// [[FBRoute POST:@"/wda/apps/launch"] respondWithTarget:self action:@selector(handleSessionAppLaunch:)]
 	data := make(map[string]interface{})
 	data["bundleId"] = bundleId
@@ -508,6 +509,7 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) {
 }
 
 func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) {
+	log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunchUnattached")
 	// [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)]
 	data := map[string]interface{}{"bundleId": bundleId}
 	_, err = wd.Session.POST(data, "/wda/apps/launchUnattached")
@@ -519,6 +521,7 @@ func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) {
 }
 
 func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error) {
+	log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppTerminate")
 	// [[FBRoute POST:@"/wda/apps/terminate"] respondWithTarget:self action:@selector(handleSessionAppTerminate:)]
 	data := map[string]interface{}{"bundleId": bundleId}
 	var rawResp DriverRawResponse
@@ -533,6 +536,7 @@ func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error)
 }
 
 func (wd *WDADriver) AppActivate(bundleId string) (err error) {
+	log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppActivate")
 	// [[FBRoute POST:@"/wda/apps/activate"] respondWithTarget:self action:@selector(handleSessionAppActivate:)]
 	data := map[string]interface{}{"bundleId": bundleId}
 	urlStr := fmt.Sprintf("/session/%s/wda/apps/activate", wd.Session.ID)
@@ -541,6 +545,7 @@ func (wd *WDADriver) AppActivate(bundleId string) (err error) {
 }
 
 func (wd *WDADriver) AppDeactivate(second float64) (err error) {
+	log.Info().Float64("second", second).Msg("WDADriver.AppDeactivate")
 	// [[FBRoute POST:@"/wda/deactivateApp"] respondWithTarget:self action:@selector(handleDeactivateAppCommand:)]
 	if second < 3 {
 		second = 3.0
@@ -576,6 +581,7 @@ func (wd *WDADriver) ForegroundInfo() (appInfo types.AppInfo, err error) {
 }
 
 func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapXY")
 	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
 	absX, absY, err := convertToAbsolutePoint(wd, x, y)
 	if err != nil {
@@ -585,9 +591,10 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY")
 	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 	data := map[string]interface{}{
 		"x": wd.toScale(x),
 		"y": wd.toScale(y),
@@ -600,6 +607,7 @@ func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
 }
 
 func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
+	log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap")
 	// [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)]
 	var err error
 	x, y, err = convertToAbsolutePoint(wd, x, y)
@@ -608,7 +616,7 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error
 	}
 
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 	x = wd.toScale(x)
 	y = wd.toScale(y)
 	data := map[string]interface{}{
@@ -622,8 +630,9 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error
 
 // FIXME: hold not work
 func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) {
+	log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TouchAndHold")
 	actionOptions := option.NewActionOptions(opts...)
-	x, y = actionOptions.ApplyOffset(x, y)
+	x, y = actionOptions.ApplyTapOffset(x, y)
 	if actionOptions.Duration == 0 {
 		opts = append(opts, option.WithPressDuration(1))
 	}
@@ -631,6 +640,8 @@ func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er
 }
 
 func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
+	log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
+		Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag")
 	// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
 	var err error
 	fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(wd, fromX, fromY, toX, toY)
@@ -641,6 +652,9 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO
 	fromY = wd.toScale(fromY)
 	toX = wd.toScale(toX)
 	toY = wd.toScale(toY)
+	actionOptions := option.NewActionOptions(opts...)
+	fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
+
 	data := map[string]interface{}{
 		"fromX": math.Round(fromX*10) / 10,
 		"fromY": math.Round(fromY*10) / 10,
@@ -689,6 +703,7 @@ func (wd *WDADriver) SetIme(ime string) error {
 }
 
 func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) {
+	log.Info().Str("text", text).Msg("WDADriver.Input")
 	// [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)]
 	data := map[string]interface{}{"value": strings.Split(text, "")}
 	option.MergeOptions(data, opts...)
@@ -698,6 +713,7 @@ func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error)
 }
 
 func (wd *WDADriver) Backspace(count int, opts ...option.ActionOption) (err error) {
+	log.Info().Int("count", count).Msg("WDADriver.Backspace")
 	if count == 0 {
 		return nil
 	}
@@ -713,6 +729,7 @@ func (wd *WDADriver) AppClear(packageName string) error {
 
 // Back simulates a short press on the BACK button.
 func (wd *WDADriver) Back() (err error) {
+	log.Info().Msg("WDADriver.Back")
 	return wd.Swipe(0, 0.5, 0.6, 0.5)
 }
 
@@ -876,6 +893,7 @@ func (wd *WDADriver) triggerWDALog(data map[string]interface{}) (rawResp []byte,
 }
 
 func (wd *WDADriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) {
+	log.Info().Msg("WDADriver.ScreenRecord")
 	timestamp := time.Now().Format("20060102_150405") + fmt.Sprintf("_%03d", time.Now().UnixNano()/1e6%1000)
 	fileName := filepath.Join(config.GetConfig().ScreenShotsPath, fmt.Sprintf("%s.mp4", timestamp))
 
@@ -951,6 +969,7 @@ func (wd *WDADriver) StartCaptureLog(identifier ...string) error {
 }
 
 func (wd *WDADriver) PushImage(localPath string) error {
+	log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage")
 	localFile, err := os.Open(localPath)
 	if err != nil {
 		return err
@@ -971,6 +990,7 @@ func (wd *WDADriver) PushImage(localPath string) error {
 }
 
 func (wd *WDADriver) ClearImages() error {
+	log.Info().Msg("WDADriver.ClearImages")
 	data := map[string]interface{}{}
 
 	_, err := wd.Session.POST(data, "/gtf/albums/clear")

uixt/option/action.go

@@ -95,13 +95,18 @@ func (o *ActionOptions) Options() []ActionOption {
 		options = append(options, WithScope(
 			o.Scope[0], o.Scope[1], o.Scope[2], o.Scope[3]))
 	}
-	if len(o.Offset) == 2 {
+	if len(o.TapOffset) == 2 {
 		// for tap [x,y] offset
-		options = append(options, WithTapOffset(o.Offset[0], o.Offset[1]))
+		options = append(options, WithTapOffset(o.TapOffset[0], o.TapOffset[1]))
-	} else if len(o.Offset) == 4 {
+	}
+	if o.TapRandomRect {
+		// tap random point in OCR/CV rectangle
+		options = append(options, WithTapRandomRect(true))
+	}
+	if len(o.SwipeOffset) == 4 {
 		// for swipe [fromX, fromY, toX, toY] offset
 		options = append(options, WithSwipeOffset(
-			o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3]))
+			o.SwipeOffset[0], o.SwipeOffset[1], o.SwipeOffset[2], o.SwipeOffset[3]))
 	}
 	if len(o.OffsetRandomRange) == 2 {
 		options = append(options, WithOffsetRandomRange(
@@ -131,17 +136,32 @@ func (o *ActionOptions) Options() []ActionOption {
 	return options
 }
 
-func (o *ActionOptions) ApplyOffset(absX, absY float64) (float64, float64) {
+func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) {
-	if len(o.Offset) == 2 {
+	if len(o.TapOffset) == 2 {
-		absX += float64(o.Offset[0])
+		absX += float64(o.TapOffset[0])
-		absY += float64(o.Offset[1])
+		absY += float64(o.TapOffset[1])
 	}
-	absX += o.GenerateRandomOffset()
+	absX += o.generateRandomOffset()
-	absY += o.GenerateRandomOffset()
+	absY += o.generateRandomOffset()
 	return absX, absY
 }
 
-func (o *ActionOptions) GenerateRandomOffset() float64 {
+func (o *ActionOptions) ApplySwipeOffset(absFromX, absFromY, absToX, absToY float64) (
+	float64, float64, float64, float64) {
+	if len(o.SwipeOffset) == 4 {
+		absFromX += float64(o.SwipeOffset[0])
+		absFromY += float64(o.SwipeOffset[1])
+		absToX += float64(o.SwipeOffset[2])
+		absToY += float64(o.SwipeOffset[3])
+	}
+	absFromX += o.generateRandomOffset()
+	absFromY += o.generateRandomOffset()
+	absToX += o.generateRandomOffset()
+	absToY += o.generateRandomOffset()
+	return absFromX, absFromY, absToX, absToY
+}
+
+func (o *ActionOptions) generateRandomOffset() float64 {
 	if len(o.OffsetRandomRange) != 2 {
 		// invalid offset random range, should be [min, max]
 		return 0
@@ -276,7 +296,7 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
 // swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY]
 func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption {
 	return func(o *ActionOptions) {
-		o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY}
+		o.SwipeOffset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY}
 	}
 }
 

uixt/option/screen.go

@@ -216,7 +216,9 @@ type ScreenFilterOptions struct {
 	AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"`
 
 	Regex               bool  `json:"regex,omitempty" yaml:"regex,omitempty"`                             // use regex to match text
-	Offset              []int `json:"offset,omitempty" yaml:"offset,omitempty"`                           // used to tap offset of point
+	TapOffset           []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"`                   // tap with absolute point offset
+	TapRandomRect       bool  `json:"tap_random_rect,omitempty" yaml:"tap_random_rect,omitempty"`         // tap random point in text/image rectangle
+	SwipeOffset         []int `json:"swipe_offset,omitempty" yaml:"swipe_offset,omitempty"`               // swipe with direction offset
 	OffsetRandomRange   []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points
 	Index               int   `json:"index,omitempty" yaml:"index,omitempty"`                             // index of the target element
 	MatchOne            bool  `json:"match_one,omitempty" yaml:"match_one,omitempty"`
@@ -242,7 +244,15 @@ func WithAbsScope(x1, y1, x2, y2 int) ActionOption {
 // tap [x, y] with offset [offsetX, offsetY]
 func WithTapOffset(offsetX, offsetY int) ActionOption {
 	return func(o *ActionOptions) {
-		o.Offset = []int{offsetX, offsetY}
+		o.TapOffset = []int{offsetX, offsetY}
+	}
+}
+
+// WithTapRandomRect is used with TapByOCR and TapByCV
+// when set true, tap random point in text/image rectangle
+func WithTapRandomRect(tapRandom bool) ActionOption {
+	return func(o *ActionOptions) {
+		o.TapRandomRect = tapRandom
 	}
 }