Merge pull request #3550 from seleniumbase/cdp-mode-patch-36

CDP Mode: Patch 36

examples/cdp_mode/ReadMe.md

@@ -328,11 +328,11 @@ with SB(uc=True, test=True, ad_block=True) as sb:
 ```python
 from seleniumbase import SB
 
-with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
+with SB(uc=True, test=True, locale_code="en", pls="none") as sb:
     url = "https://www.nike.com/"
     sb.activate_cdp_mode(url)
     sb.sleep(2.5)
-    sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
+    sb.cdp.mouse_click('div[data-testid="user-tools-container"]')
     sb.sleep(1.5)
     search = "Nike Air Force 1"
     sb.cdp.press_keys('input[type="search"]', search)
@@ -470,6 +470,8 @@ sb.cdp.is_exact_text_visible(text, selector="body")
 sb.cdp.wait_for_text(text, selector="body", timeout=None)
 sb.cdp.wait_for_text_not_visible(text, selector="body", timeout=None)
 sb.cdp.wait_for_element_visible(selector, timeout=None)
+sb.cdp.wait_for_element_not_visible(selector, timeout=None)
+sb.cdp.wait_for_element_absent(selector, timeout=None)
 sb.cdp.assert_element(selector, timeout=None)
 sb.cdp.assert_element_visible(selector, timeout=None)
 sb.cdp.assert_element_present(selector, timeout=None)

examples/cdp_mode/raw_cdp_nike.py

@@ -8,7 +8,7 @@ driver = cdp_driver.cdp_util.start_sync()
 page = loop.run_until_complete(driver.get(url))
 sb = sb_cdp.CDPMethods(loop, page, driver)
 
-search = "Nike Fly Shoes"
+search = "Road Racing Shoes"
 sb.click('div[data-testid="user-tools-container"]')
 sb.sleep(1)
 sb.press_keys('input[type="search"]', search)

examples/cdp_mode/raw_footlocker.py

@@ -5,9 +5,9 @@ with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
     sb.activate_cdp_mode(url)
     sb.sleep(2.5)
     sb.cdp.click_if_visible('button[id*="Agree"]')
-    sb.sleep(2.5)
+    sb.sleep(1.5)
     sb.cdp.mouse_click('input[aria-label="Search"]')
-    sb.sleep(2.5)
+    sb.sleep(1.5)
     search = "Nike Shoes"
     sb.cdp.press_keys('input[aria-label="Search"]', search)
     sb.sleep(2.5)

examples/cdp_mode/raw_nike.py

@@ -1,10 +1,10 @@
 from seleniumbase import SB
 
-with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
+with SB(uc=True, test=True, locale_code="en", pls="none") as sb:
     url = "https://www.nike.com/"
     sb.activate_cdp_mode(url)
     sb.sleep(2.5)
-    sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
+    sb.cdp.mouse_click('div[data-testid="user-tools-container"]')
     sb.sleep(1.5)
     search = "Nike Air Force 1"
     sb.cdp.press_keys('input[type="search"]', search)

examples/cdp_mode/raw_res_nike.py

@@ -25,13 +25,13 @@ async def receive_handler(event: mycdp.network.ResponseReceived):
     print(event.response)
 
 
-with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
+with SB(uc=True, test=True, locale_code="en", pls="none") as sb:
     url = "https://www.nike.com/"
     sb.activate_cdp_mode(url)
     sb.cdp.add_handler(mycdp.network.RequestWillBeSent, send_handler)
     sb.cdp.add_handler(mycdp.network.ResponseReceived, receive_handler)
     sb.sleep(2.5)
-    sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
+    sb.cdp.mouse_click('div[data-testid="user-tools-container"]')
     sb.sleep(1.5)
     search = "Nike Air Force 1"
     sb.cdp.press_keys('input[type="search"]', search)

examples/cdp_mode/raw_science.py

@@ -0,0 +1,15 @@
+from seleniumbase import SB
+
+with SB(uc=True, incognito=True, test=True) as sb:
+    url = "https://earth.esa.int/eogateway/search"
+    sb.activate_cdp_mode(url)
+    sb.sleep(1)
+    sb.cdp.click_if_visible('button:contains("Accept cookies")')
+    for i in range(20):
+        sb.cdp.scroll_to_bottom()
+        sb.cdp.click_if_visible('button:contains("READ MORE")')
+    sb.sleep(1)
+    elements = sb.cdp.find_elements("h4 a span")
+    for element in elements:
+        print(element.text)
+    print("*** Total entries: %s" % len(elements))

examples/presenter/uc_presentation_4.py

@@ -838,11 +838,11 @@ class UCPresentationClass(BaseCase):
         )
         self.begin_presentation(filename="uc_presentation.html")
 
-        with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
+        with SB(uc=True, test=True, locale_code="en", pls="none") as sb:
             url = "https://www.nike.com/"
             sb.activate_cdp_mode(url)
             sb.sleep(2.5)
-            sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
+            sb.cdp.mouse_click('div[data-testid="user-tools-container"]')
             sb.sleep(1.5)
             search = "Nike Air Force 1"
             sb.cdp.press_keys('input[type="search"]', search)

requirements.txt

@@ -43,7 +43,7 @@ trio-websocket==0.12.1
 wsproto==1.2.0
 websocket-client==1.8.0
 selenium==4.27.1;python_version<"3.9"
-selenium==4.28.1;python_version>="3.9"
+selenium==4.29.0;python_version>="3.9"
 cssselect==1.2.0
 sortedcontainers==2.4.0
 execnet==2.1.1

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.34.17"
+__version__ = "4.35.0"

seleniumbase/core/browser_launcher.py

@@ -723,6 +723,8 @@ def uc_open_with_cdp_mode(driver, url=None):
     cdp.wait_for_text = CDPM.wait_for_text
     cdp.wait_for_text_not_visible = CDPM.wait_for_text_not_visible
     cdp.wait_for_element_visible = CDPM.wait_for_element_visible
+    cdp.wait_for_element_not_visible = CDPM.wait_for_element_not_visible
+    cdp.wait_for_element_absent = CDPM.wait_for_element_absent
     cdp.assert_element = CDPM.assert_element
     cdp.assert_element_visible = CDPM.assert_element_visible
     cdp.assert_element_present = CDPM.assert_element_present
@@ -1628,9 +1630,19 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None):
         ):
             driver.uc_open_with_disconnect(driver.current_url, 3.8)
             with suppress(Exception):
+                if "--debug" in sys.argv:
+                    if sb_config._saved_cf_tab_count == 1:
+                        print(' <DEBUG> pyautogui.press("\\t")')
+                    else:
+                        print(
+                            ' <DEBUG> pyautogui.press("\\t") * %s'
+                            % sb_config._saved_cf_tab_count
+                        )
                 for i in range(sb_config._saved_cf_tab_count):
                     pyautogui.press("\t")
                     time.sleep(0.027)
+                if "--debug" in sys.argv:
+                    print(' <DEBUG> pyautogui.press(" ")')
                 pyautogui.press(" ")
         else:
             driver.disconnect()
@@ -2310,7 +2322,14 @@ def _set_chrome_options(
         and not enable_3d_apis
     ):
         chrome_options.add_argument("--disable-gpu")
-    if not IS_LINUX and is_using_uc(undetectable, browser_name):
+    if (
+        (not IS_LINUX and is_using_uc(undetectable, browser_name))
+        or (
+            IS_MAC
+            and binary_location
+            and "chrome-headless-shell" in binary_location
+        )
+    ):
         chrome_options.add_argument("--disable-dev-shm-usage")
         chrome_options.add_argument("--disable-application-cache")
     if IS_LINUX:

seleniumbase/core/sb_cdp.py

@@ -497,7 +497,7 @@ class CDPMethods():
             element.send_keys("\r\n")
             time.sleep(0.044)
         self.__slow_mode_pause_if_set()
-        return self.loop.run_until_complete(self.page.wait())
+        return self.loop.run_until_complete(self.page.sleep(0.025))
 
     def __query_selector(self, element, selector):
         selector = self.__convert_to_css_if_xpath(selector)
@@ -864,7 +864,7 @@ class CDPMethods():
             text = text[:-1] + "\r\n"
         element.send_keys(text)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def press_keys(self, selector, text, timeout=None):
         """Similar to send_keys(), but presses keys at human speed."""
@@ -884,7 +884,7 @@ class CDPMethods():
             element.send_keys("\r\n")
             time.sleep(0.044)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def type(self, selector, text, timeout=None):
         """Similar to send_keys(), but clears the text field first."""
@@ -899,7 +899,7 @@ class CDPMethods():
             text = text[:-1] + "\r\n"
         element.send_keys(text)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def set_value(self, selector, text, timeout=None):
         """Similar to send_keys(), but clears the text field first."""
@@ -937,7 +937,7 @@ class CDPMethods():
             self.__add_light_pause()
             self.send_keys(selector, "\n")
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def evaluate(self, expression):
         """Run a JavaScript expression and return the result."""
@@ -1377,7 +1377,7 @@ class CDPMethods():
             pyautogui.press(key)
             time.sleep(0.044)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def gui_press_keys(self, keys):
         self.__install_pyautogui_if_missing()
@@ -1392,7 +1392,7 @@ class CDPMethods():
                 pyautogui.press(key)
                 time.sleep(0.044)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def gui_write(self, text):
         self.__install_pyautogui_if_missing()
@@ -1405,7 +1405,7 @@ class CDPMethods():
             self.__make_sure_pyautogui_lock_is_writable()
             pyautogui.write(text)
         self.__slow_mode_pause_if_set()
-        self.loop.run_until_complete(self.page.wait())
+        self.loop.run_until_complete(self.page.sleep(0.025))
 
     def __gui_click_x_y(self, x, y, timeframe=0.25, uc_lock=False):
         self.__install_pyautogui_if_missing()
@@ -1820,6 +1820,50 @@ class CDPMethods():
             time.sleep(0.1)
         raise Exception("Element {%s} was not visible!" % selector)
 
+    def wait_for_element_not_visible(self, selector, timeout=None):
+        """Wait for element to not be visible on page. (May still be in DOM)"""
+        if not timeout:
+            timeout = settings.SMALL_TIMEOUT
+        start_ms = time.time() * 1000.0
+        stop_ms = start_ms + (timeout * 1000.0)
+        for i in range(int(timeout * 10)):
+            if not self.is_element_present(selector):
+                return True
+            elif not self.is_element_visible(selector):
+                return True
+            now_ms = time.time() * 1000.0
+            if now_ms >= stop_ms:
+                break
+            time.sleep(0.1)
+        plural = "s"
+        if timeout == 1:
+            plural = ""
+        raise Exception(
+            "Element {%s} was still visible after %s second%s!"
+            % (selector, timeout, plural)
+        )
+
+    def wait_for_element_absent(self, selector, timeout=None):
+        """Wait for element to not be present in the DOM."""
+        if not timeout:
+            timeout = settings.SMALL_TIMEOUT
+        start_ms = time.time() * 1000.0
+        stop_ms = start_ms + (timeout * 1000.0)
+        for i in range(int(timeout * 10)):
+            if not self.is_element_present(selector):
+                return True
+            now_ms = time.time() * 1000.0
+            if now_ms >= stop_ms:
+                break
+            time.sleep(0.1)
+        plural = "s"
+        if timeout == 1:
+            plural = ""
+        raise Exception(
+            "Element {%s} was still present after %s second%s!"
+            % (selector, timeout, plural)
+        )
+
     def assert_element(self, selector, timeout=None):
         """Same as assert_element_visible()"""
         self.assert_element_visible(selector, timeout=timeout)
@@ -1851,47 +1895,13 @@ class CDPMethods():
 
     def assert_element_absent(self, selector, timeout=None):
         """Assert element is not present in the DOM."""
-        if not timeout:
+        self.wait_for_element_absent(selector, timeout=timeout)
-            timeout = settings.SMALL_TIMEOUT
+        return True
-        start_ms = time.time() * 1000.0
-        stop_ms = start_ms + (timeout * 1000.0)
-        for i in range(int(timeout * 10)):
-            if not self.is_element_present(selector):
-                return True
-            now_ms = time.time() * 1000.0
-            if now_ms >= stop_ms:
-                break
-            time.sleep(0.1)
-        plural = "s"
-        if timeout == 1:
-            plural = ""
-        raise Exception(
-            "Element {%s} was still present after %s second%s!"
-            % (selector, timeout, plural)
-        )
 
     def assert_element_not_visible(self, selector, timeout=None):
         """Assert element is not visible on page. (May still be in DOM)"""
-        if not timeout:
+        self.wait_for_element_not_visible(selector, timeout=timeout)
-            timeout = settings.SMALL_TIMEOUT
+        return True
-        start_ms = time.time() * 1000.0
-        stop_ms = start_ms + (timeout * 1000.0)
-        for i in range(int(timeout * 10)):
-            if not self.is_element_present(selector):
-                return True
-            elif not self.is_element_visible(selector):
-                return True
-            now_ms = time.time() * 1000.0
-            if now_ms >= stop_ms:
-                break
-            time.sleep(0.1)
-        plural = "s"
-        if timeout == 1:
-            plural = ""
-        raise Exception(
-            "Element {%s} was still visible after %s second%s!"
-            % (selector, timeout, plural)
-        )
 
     def assert_element_attribute(self, selector, attribute, value=None):
         attributes = self.get_element_attributes(selector)

seleniumbase/fixtures/base_case.py

@@ -9124,7 +9124,7 @@ class BaseCase(unittest.TestCase):
         original_selector = selector
         selector, by = self.__recalculate_selector(selector, by)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element_absent(selector)
+            self.cdp.wait_for_element_absent(selector, timeout=timeout)
             return True
         return page_actions.wait_for_element_absent(
             self.driver,
@@ -9585,7 +9585,7 @@ class BaseCase(unittest.TestCase):
             self.assert_elements_present(selector, by=by, timeout=timeout)
             return True
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element_present(selector)
+            self.cdp.assert_element_present(selector, timeout=timeout)
             return True
         if self.__is_shadow_selector(selector):
             self.__assert_shadow_element_present(selector)
@@ -9662,7 +9662,7 @@ class BaseCase(unittest.TestCase):
         if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
             timeout = self.__get_new_timeout(timeout)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element(selector)
+            self.cdp.assert_element(selector, timeout=timeout)
             return True
         if isinstance(selector, list):
             self.assert_elements(selector, by=by, timeout=timeout)
@@ -9955,7 +9955,7 @@ class BaseCase(unittest.TestCase):
                         messenger_post, selector, by
                     )
         elif self.__is_cdp_swap_needed():
-            self.cdp.assert_text(text, selector)
+            self.cdp.assert_text(text, selector, timeout=timeout)
             return True
         elif not self.is_connected():
             self.connect()
@@ -10005,7 +10005,7 @@ class BaseCase(unittest.TestCase):
         original_selector = selector
         selector, by = self.__recalculate_selector(selector, by)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_exact_text(text, selector)
+            self.cdp.assert_exact_text(text, selector, timeout=timeout)
             return True
         if self.__is_shadow_selector(selector):
             self.__assert_exact_shadow_text_visible(text, selector, timeout)
@@ -10245,6 +10245,9 @@ class BaseCase(unittest.TestCase):
             timeout = self.__get_new_timeout(timeout)
         original_selector = selector
         selector, by = self.__recalculate_selector(selector, by)
+        if self.__is_cdp_swap_needed():
+            self.cdp.wait_for_element_absent(selector, timeout=timeout)
+            return True
         return page_actions.wait_for_element_absent(
             self.driver,
             selector,
@@ -10267,7 +10270,7 @@ class BaseCase(unittest.TestCase):
         if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
             timeout = self.__get_new_timeout(timeout)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element_absent(selector)
+            self.cdp.assert_element_absent(selector, timeout=timeout)
             return True
         self.wait_for_element_absent(selector, by=by, timeout=timeout)
         return True
@@ -10288,7 +10291,7 @@ class BaseCase(unittest.TestCase):
         original_selector = selector
         selector, by = self.__recalculate_selector(selector, by)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element_not_visible(selector)
+            self.cdp.wait_for_element_not_visible(selector, timeout=timeout)
             return True
         return page_actions.wait_for_element_not_visible(
             self.driver,
@@ -10310,7 +10313,7 @@ class BaseCase(unittest.TestCase):
         if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
             timeout = self.__get_new_timeout(timeout)
         if self.__is_cdp_swap_needed():
-            self.cdp.assert_element_not_visible(selector)
+            self.cdp.assert_element_not_visible(selector, timeout=timeout)
             return True
         self.wait_for_element_not_visible(selector, by=by, timeout=timeout)
         if self.recorder_mode and self.__current_url_is_recordable():

setup.py

@@ -192,7 +192,7 @@ setup(
         'wsproto==1.2.0',
         'websocket-client==1.8.0',
         'selenium==4.27.1;python_version<"3.9"',
-        'selenium==4.28.1;python_version>="3.9"',
+        'selenium==4.29.0;python_version>="3.9"',
         'cssselect==1.2.0',
         "sortedcontainers==2.4.0",
         'execnet==2.1.1',