From 24317c15fbe1ea640ff70a938468fbc85223dccd Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:50:04 -0500 Subject: [PATCH 1/4] Update CDP Mode --- examples/cdp_mode/ReadMe.md | 2 + seleniumbase/core/browser_launcher.py | 21 +++++- seleniumbase/core/sb_cdp.py | 102 ++++++++++++++------------ seleniumbase/fixtures/base_case.py | 19 +++-- 4 files changed, 89 insertions(+), 55 deletions(-) diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 97b7edec..31d4bdea 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -470,6 +470,8 @@ sb.cdp.is_exact_text_visible(text, selector="body") sb.cdp.wait_for_text(text, selector="body", timeout=None) sb.cdp.wait_for_text_not_visible(text, selector="body", timeout=None) sb.cdp.wait_for_element_visible(selector, timeout=None) +sb.cdp.wait_for_element_not_visible(selector, timeout=None) +sb.cdp.wait_for_element_absent(selector, timeout=None) sb.cdp.assert_element(selector, timeout=None) sb.cdp.assert_element_visible(selector, timeout=None) sb.cdp.assert_element_present(selector, timeout=None) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 4fac3296..79fb720f 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -723,6 +723,8 @@ def uc_open_with_cdp_mode(driver, url=None): cdp.wait_for_text = CDPM.wait_for_text cdp.wait_for_text_not_visible = CDPM.wait_for_text_not_visible cdp.wait_for_element_visible = CDPM.wait_for_element_visible + cdp.wait_for_element_not_visible = CDPM.wait_for_element_not_visible + cdp.wait_for_element_absent = CDPM.wait_for_element_absent cdp.assert_element = CDPM.assert_element cdp.assert_element_visible = CDPM.assert_element_visible cdp.assert_element_present = CDPM.assert_element_present @@ -1628,9 +1630,19 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None): ): driver.uc_open_with_disconnect(driver.current_url, 3.8) with suppress(Exception): + if "--debug" in sys.argv: + if sb_config._saved_cf_tab_count == 1: + print(' pyautogui.press("\\t")') + else: + print( + ' pyautogui.press("\\t") * %s' + % sb_config._saved_cf_tab_count + ) for i in range(sb_config._saved_cf_tab_count): pyautogui.press("\t") time.sleep(0.027) + if "--debug" in sys.argv: + print(' pyautogui.press(" ")') pyautogui.press(" ") else: driver.disconnect() @@ -2310,7 +2322,14 @@ def _set_chrome_options( and not enable_3d_apis ): chrome_options.add_argument("--disable-gpu") - if not IS_LINUX and is_using_uc(undetectable, browser_name): + if ( + (not IS_LINUX and is_using_uc(undetectable, browser_name)) + or ( + IS_MAC + and binary_location + and "chrome-headless-shell" in binary_location + ) + ): chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--disable-application-cache") if IS_LINUX: diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 2b0003df..d8388022 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -497,7 +497,7 @@ class CDPMethods(): element.send_keys("\r\n") time.sleep(0.044) self.__slow_mode_pause_if_set() - return self.loop.run_until_complete(self.page.wait()) + return self.loop.run_until_complete(self.page.sleep(0.025)) def __query_selector(self, element, selector): selector = self.__convert_to_css_if_xpath(selector) @@ -864,7 +864,7 @@ class CDPMethods(): text = text[:-1] + "\r\n" element.send_keys(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def press_keys(self, selector, text, timeout=None): """Similar to send_keys(), but presses keys at human speed.""" @@ -884,7 +884,7 @@ class CDPMethods(): element.send_keys("\r\n") time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def type(self, selector, text, timeout=None): """Similar to send_keys(), but clears the text field first.""" @@ -899,7 +899,7 @@ class CDPMethods(): text = text[:-1] + "\r\n" element.send_keys(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def set_value(self, selector, text, timeout=None): """Similar to send_keys(), but clears the text field first.""" @@ -937,7 +937,7 @@ class CDPMethods(): self.__add_light_pause() self.send_keys(selector, "\n") self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def evaluate(self, expression): """Run a JavaScript expression and return the result.""" @@ -1377,7 +1377,7 @@ class CDPMethods(): pyautogui.press(key) time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def gui_press_keys(self, keys): self.__install_pyautogui_if_missing() @@ -1392,7 +1392,7 @@ class CDPMethods(): pyautogui.press(key) time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def gui_write(self, text): self.__install_pyautogui_if_missing() @@ -1405,7 +1405,7 @@ class CDPMethods(): self.__make_sure_pyautogui_lock_is_writable() pyautogui.write(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def __gui_click_x_y(self, x, y, timeframe=0.25, uc_lock=False): self.__install_pyautogui_if_missing() @@ -1820,6 +1820,50 @@ class CDPMethods(): time.sleep(0.1) raise Exception("Element {%s} was not visible!" % selector) + def wait_for_element_not_visible(self, selector, timeout=None): + """Wait for element to not be visible on page. (May still be in DOM)""" + if not timeout: + timeout = settings.SMALL_TIMEOUT + start_ms = time.time() * 1000.0 + stop_ms = start_ms + (timeout * 1000.0) + for i in range(int(timeout * 10)): + if not self.is_element_present(selector): + return True + elif not self.is_element_visible(selector): + return True + now_ms = time.time() * 1000.0 + if now_ms >= stop_ms: + break + time.sleep(0.1) + plural = "s" + if timeout == 1: + plural = "" + raise Exception( + "Element {%s} was still visible after %s second%s!" + % (selector, timeout, plural) + ) + + def wait_for_element_absent(self, selector, timeout=None): + """Wait for element to not be present in the DOM.""" + if not timeout: + timeout = settings.SMALL_TIMEOUT + start_ms = time.time() * 1000.0 + stop_ms = start_ms + (timeout * 1000.0) + for i in range(int(timeout * 10)): + if not self.is_element_present(selector): + return True + now_ms = time.time() * 1000.0 + if now_ms >= stop_ms: + break + time.sleep(0.1) + plural = "s" + if timeout == 1: + plural = "" + raise Exception( + "Element {%s} was still present after %s second%s!" + % (selector, timeout, plural) + ) + def assert_element(self, selector, timeout=None): """Same as assert_element_visible()""" self.assert_element_visible(selector, timeout=timeout) @@ -1851,47 +1895,13 @@ class CDPMethods(): def assert_element_absent(self, selector, timeout=None): """Assert element is not present in the DOM.""" - if not timeout: - timeout = settings.SMALL_TIMEOUT - start_ms = time.time() * 1000.0 - stop_ms = start_ms + (timeout * 1000.0) - for i in range(int(timeout * 10)): - if not self.is_element_present(selector): - return True - now_ms = time.time() * 1000.0 - if now_ms >= stop_ms: - break - time.sleep(0.1) - plural = "s" - if timeout == 1: - plural = "" - raise Exception( - "Element {%s} was still present after %s second%s!" - % (selector, timeout, plural) - ) + self.wait_for_element_absent(selector, timeout=timeout) + return True def assert_element_not_visible(self, selector, timeout=None): """Assert element is not visible on page. (May still be in DOM)""" - if not timeout: - timeout = settings.SMALL_TIMEOUT - start_ms = time.time() * 1000.0 - stop_ms = start_ms + (timeout * 1000.0) - for i in range(int(timeout * 10)): - if not self.is_element_present(selector): - return True - elif not self.is_element_visible(selector): - return True - now_ms = time.time() * 1000.0 - if now_ms >= stop_ms: - break - time.sleep(0.1) - plural = "s" - if timeout == 1: - plural = "" - raise Exception( - "Element {%s} was still visible after %s second%s!" - % (selector, timeout, plural) - ) + self.wait_for_element_not_visible(selector, timeout=timeout) + return True def assert_element_attribute(self, selector, attribute, value=None): attributes = self.get_element_attributes(selector) diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 8c6c57d0..5a1dd8c5 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -9124,7 +9124,7 @@ class BaseCase(unittest.TestCase): original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_absent(selector) + self.cdp.wait_for_element_absent(selector, timeout=timeout) return True return page_actions.wait_for_element_absent( self.driver, @@ -9585,7 +9585,7 @@ class BaseCase(unittest.TestCase): self.assert_elements_present(selector, by=by, timeout=timeout) return True if self.__is_cdp_swap_needed(): - self.cdp.assert_element_present(selector) + self.cdp.assert_element_present(selector, timeout=timeout) return True if self.__is_shadow_selector(selector): self.__assert_shadow_element_present(selector) @@ -9662,7 +9662,7 @@ class BaseCase(unittest.TestCase): if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element(selector) + self.cdp.assert_element(selector, timeout=timeout) return True if isinstance(selector, list): self.assert_elements(selector, by=by, timeout=timeout) @@ -9955,7 +9955,7 @@ class BaseCase(unittest.TestCase): messenger_post, selector, by ) elif self.__is_cdp_swap_needed(): - self.cdp.assert_text(text, selector) + self.cdp.assert_text(text, selector, timeout=timeout) return True elif not self.is_connected(): self.connect() @@ -10005,7 +10005,7 @@ class BaseCase(unittest.TestCase): original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_exact_text(text, selector) + self.cdp.assert_exact_text(text, selector, timeout=timeout) return True if self.__is_shadow_selector(selector): self.__assert_exact_shadow_text_visible(text, selector, timeout) @@ -10245,6 +10245,9 @@ class BaseCase(unittest.TestCase): timeout = self.__get_new_timeout(timeout) original_selector = selector selector, by = self.__recalculate_selector(selector, by) + if self.__is_cdp_swap_needed(): + self.cdp.wait_for_element_absent(selector, timeout=timeout) + return True return page_actions.wait_for_element_absent( self.driver, selector, @@ -10267,7 +10270,7 @@ class BaseCase(unittest.TestCase): if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_absent(selector) + self.cdp.assert_element_absent(selector, timeout=timeout) return True self.wait_for_element_absent(selector, by=by, timeout=timeout) return True @@ -10288,7 +10291,7 @@ class BaseCase(unittest.TestCase): original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_not_visible(selector) + self.cdp.wait_for_element_not_visible(selector, timeout=timeout) return True return page_actions.wait_for_element_not_visible( self.driver, @@ -10310,7 +10313,7 @@ class BaseCase(unittest.TestCase): if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_not_visible(selector) + self.cdp.assert_element_not_visible(selector, timeout=timeout) return True self.wait_for_element_not_visible(selector, by=by, timeout=timeout) if self.recorder_mode and self.__current_url_is_recordable(): From 0949787a392daf033beb4ecb3ce2ef86ae82deb0 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:52:35 -0500 Subject: [PATCH 2/4] Update CDP Mode examples --- examples/cdp_mode/ReadMe.md | 4 ++-- examples/cdp_mode/raw_cdp_nike.py | 2 +- examples/cdp_mode/raw_footlocker.py | 4 ++-- examples/cdp_mode/raw_nike.py | 4 ++-- examples/cdp_mode/raw_res_nike.py | 4 ++-- examples/cdp_mode/raw_science.py | 15 +++++++++++++++ examples/presenter/uc_presentation_4.py | 4 ++-- 7 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 examples/cdp_mode/raw_science.py diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 31d4bdea..2186fa90 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -328,11 +328,11 @@ with SB(uc=True, test=True, ad_block=True) as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_cdp_nike.py b/examples/cdp_mode/raw_cdp_nike.py index 48d83563..88c72539 100644 --- a/examples/cdp_mode/raw_cdp_nike.py +++ b/examples/cdp_mode/raw_cdp_nike.py @@ -8,7 +8,7 @@ driver = cdp_driver.cdp_util.start_sync() page = loop.run_until_complete(driver.get(url)) sb = sb_cdp.CDPMethods(loop, page, driver) -search = "Nike Fly Shoes" +search = "Road Racing Shoes" sb.click('div[data-testid="user-tools-container"]') sb.sleep(1) sb.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_footlocker.py b/examples/cdp_mode/raw_footlocker.py index 717a1e48..311d9148 100644 --- a/examples/cdp_mode/raw_footlocker.py +++ b/examples/cdp_mode/raw_footlocker.py @@ -5,9 +5,9 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('button[id*="Agree"]') - sb.sleep(2.5) + sb.sleep(1.5) sb.cdp.mouse_click('input[aria-label="Search"]') - sb.sleep(2.5) + sb.sleep(1.5) search = "Nike Shoes" sb.cdp.press_keys('input[aria-label="Search"]', search) sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_nike.py b/examples/cdp_mode/raw_nike.py index 3a811b0d..009f723d 100644 --- a/examples/cdp_mode/raw_nike.py +++ b/examples/cdp_mode/raw_nike.py @@ -1,10 +1,10 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_res_nike.py b/examples/cdp_mode/raw_res_nike.py index 416a001a..459b143e 100644 --- a/examples/cdp_mode/raw_res_nike.py +++ b/examples/cdp_mode/raw_res_nike.py @@ -25,13 +25,13 @@ async def receive_handler(event: mycdp.network.ResponseReceived): print(event.response) -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.cdp.add_handler(mycdp.network.RequestWillBeSent, send_handler) sb.cdp.add_handler(mycdp.network.ResponseReceived, receive_handler) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_science.py b/examples/cdp_mode/raw_science.py new file mode 100644 index 00000000..792dca7c --- /dev/null +++ b/examples/cdp_mode/raw_science.py @@ -0,0 +1,15 @@ +from seleniumbase import SB + +with SB(uc=True, incognito=True, test=True) as sb: + url = "https://earth.esa.int/eogateway/search" + sb.activate_cdp_mode(url) + sb.sleep(1) + sb.cdp.click_if_visible('button:contains("Accept cookies")') + for i in range(20): + sb.cdp.scroll_to_bottom() + sb.cdp.click_if_visible('button:contains("READ MORE")') + sb.sleep(1) + elements = sb.cdp.find_elements("h4 a span") + for element in elements: + print(element.text) + print("*** Total entries: %s" % len(elements)) diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index dbba53da..42ff12b3 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -838,11 +838,11 @@ class UCPresentationClass(BaseCase): ) self.begin_presentation(filename="uc_presentation.html") - with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: + with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) From 9f47f5ff425ab3e034d47daf84b6fd7308ea2aa5 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:53:08 -0500 Subject: [PATCH 3/4] Upgrade Selenium --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8763830f..1a0f763c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ trio-websocket==0.12.1 wsproto==1.2.0 websocket-client==1.8.0 selenium==4.27.1;python_version<"3.9" -selenium==4.28.1;python_version>="3.9" +selenium==4.29.0;python_version>="3.9" cssselect==1.2.0 sortedcontainers==2.4.0 execnet==2.1.1 diff --git a/setup.py b/setup.py index e4890264..dbcf8eee 100755 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ setup( 'wsproto==1.2.0', 'websocket-client==1.8.0', 'selenium==4.27.1;python_version<"3.9"', - 'selenium==4.28.1;python_version>="3.9"', + 'selenium==4.29.0;python_version>="3.9"', 'cssselect==1.2.0', "sortedcontainers==2.4.0", 'execnet==2.1.1', From 4974aeff8a5380c317325e001add4babd51e0d3d Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:53:49 -0500 Subject: [PATCH 4/4] Version 4.35.0 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index ae694d9e..ff2fa154 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.34.17" +__version__ = "4.35.0"