Merge pull request #4286 from seleniumbase/cdp-mode-patch-98

CDP Mode: Patch 98

Author: mdmintz

README.md

@@ -9,7 +9,7 @@
 
 <h1>SeleniumBase</h1>
 
-<p align="center"><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/other/sbase_text_logo3t.png" alt="SeleniumBase" title="SeleniumBase" width="630" /></a></p>
+<p align="center"><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/other/sbase_text_logo3t.png" alt="SeleniumBase" title="SeleniumBase" width="634" /></a></p>
 
 <p align="center" class="hero__title"><b>All-in-one Browser Automation Framework:<br />Web Crawling / Testing / Scraping / Stealth</b></p>
 
@@ -54,11 +54,11 @@
 
 📊 <a href="https://github.com/seleniumbase/SeleniumBase/"><b translate="no">SeleniumBase</b></a> is a complete framework for web automation,  testing, scraping, and stealth. Includes a <a href="https://docs.pytest.org/en/latest/how-to/usage.html">pytest</a> integration for customizing automation from the command-line.
 
-👤 Stealth modes: <a translate="no" href="https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/uc_mode.md">UC Mode</a> and <a translate="no" href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md"><b>CDP Mode</b></a> can bypass bot-detection, solve CAPTCHAs, and call methods from the <a href="https://chromedevtools.github.io/devtools-protocol/" translate="no">Chrome Devtools Protocol</a>. Includes <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/playwright/ReadMe.md"><b><span translate="no">Stealthy Playwright Mode</span></b></a>, which makes Playwright stealthy via CDP Mode.
+🐙 Stealth modes: <a translate="no" href="https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/uc_mode.md">UC Mode</a> and <a translate="no" href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md"><b>CDP Mode</b></a> can bypass bot-detection, solve CAPTCHAs, and call methods from the <a href="https://chromedevtools.github.io/devtools-protocol/" translate="no">Chrome Devtools Protocol</a>. Includes <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/playwright/ReadMe.md"><b><span translate="no">Stealthy Playwright Mode</span></b></a>, which makes Playwright stealthy via CDP Mode.
 
 📚 Example scripts and tests are located in [**SeleniumBase/examples/**](https://github.com/seleniumbase/SeleniumBase/tree/master/examples).
 
-👤 Stealthy example scripts are located in [**SeleniumBase/examples/cdp_mode/**](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode).
+🥷 Stealthy example scripts are located in [**SeleniumBase/examples/cdp_mode/**](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode).
 
 --------
 

examples/cdp_mode/raw_stopandshop.py

@@ -1,7 +1,7 @@
 """Test Stop & Shop search. Non-US IPs might be blocked."""
 from seleniumbase import SB
 
-with SB(uc=True, test=True, ad_block=True) as sb:
+with SB(uc=True, test=True, incognito=True) as sb:
     url = "https://stopandshop.com/"
     sb.activate_cdp_mode(url)
     sb.sleep(2.6)

requirements.txt

@@ -11,7 +11,7 @@ websockets>=16.0;python_version>="3.10"
 filelock~=3.19.1;python_version<"3.10"
 filelock>=3.25.2;python_version>="3.10"
 fasteners>=0.20
-mycdp>=1.3.4
+mycdp>=1.3.6
 pynose>=1.5.5
 platformdirs~=4.4.0;python_version<"3.10"
 platformdirs>=4.9.4;python_version>="3.10"
@@ -29,7 +29,7 @@ pyreadline3>=3.5.4;platform_system=="Windows"
 tabcompleter>=1.4.0
 pdbp>=1.8.2
 idna>=3.11
-charset-normalizer>=3.4.5,<4
+charset-normalizer>=3.4.6,<4
 urllib3>=1.26.20,<2;python_version<"3.10"
 urllib3>=1.26.20,<3;python_version>="3.10"
 requests~=2.32.5
@@ -77,7 +77,7 @@ rich>=14.3.3,<15
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage>=7.10.7;python_version<"3.10"
-coverage>=7.13.4;python_version>="3.10"
+coverage>=7.13.5;python_version>="3.10"
 pytest-cov>=7.0.0
 flake8==7.3.0
 mccabe==0.7.0

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.47.3"
+__version__ = "4.47.4"

seleniumbase/core/sb_cdp.py

@@ -391,7 +391,7 @@ def select(self, selector, timeout=None):
 
     def select_all(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         self.__add_light_pause()
         selector = self.__convert_to_css_if_xpath(selector)
         if not self.is_element_present(selector):
@@ -414,12 +414,12 @@ def select_all(self, selector, timeout=None):
 
     def find_elements(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         return self.select_all(selector, timeout=timeout)
 
     def find_visible_elements(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         visible_elements = []
         elements = self.select_all(selector, timeout=timeout)
         for element in elements:

seleniumbase/undetected/cdp_driver/browser.py

@@ -39,22 +39,30 @@ def deconstruct_browser():
     for _ in __registered__instances__:
         if not _.stopped:
             _.stop(deconstruct=True)
-        for attempt in range(5):
+        max_attempts = 5
+        for attempt in range(max_attempts):
             try:
                 if _.config and not _.config.uses_custom_data_dir:
-                    shutil.rmtree(_.config.user_data_dir, ignore_errors=False)
+                    if os.path.exists(_.config.user_data_dir):
+                        shutil.rmtree(
+                            _.config.user_data_dir, ignore_errors=False
+                        )
+                    if not os.path.exists(_.config.user_data_dir):
+                        break
+                    else:
+                        time.sleep(0.12)
             except FileNotFoundError:
                 break
             except (PermissionError, OSError) as e:
-                if attempt == 4:
+                if attempt == max_attempts - 1:
                     logger.debug(
                         "Problem removing data dir %s\n"
                         "Consider checking whether it's there "
                         "and remove it by hand\nerror: %s"
                         % (_.config.user_data_dir, e)
                     )
                     break
-                time.sleep(0.15)
+                time.sleep(0.12)
                 continue
         logging.debug("Temp profile %s was removed." % _.config.user_data_dir)
 
@@ -207,7 +215,8 @@ async def _handle_target_update(
             target_info = event.target_info
             current_tab = next(
                 filter(
-                    lambda item: item.target_id == target_info.target_id, self.targets  # noqa
+                    lambda item: item.target_id == target_info.target_id,
+                    self.targets,
                 )
             )
             current_target = current_tab.target
@@ -583,11 +592,11 @@ async def start(self=None) -> Browser:
                         else "c:/path/to/your/browser.exe"
                     )
                 )
-        if getattr(self.config, "_extensions", None):  # noqa
+        if getattr(self.config, "_extensions", None):
             self.config.add_argument(
                 "--load-extension=%s"
                 % ",".join(str(_) for _ in self.config._extensions)
-            )  # noqa
+            )
         exe = self.config.browser_executable_path
         params = self.config()
         logger.debug(
@@ -613,16 +622,17 @@ async def start(self=None) -> Browser:
         await asyncio.sleep(0.05)
         get_registered_instances().add(self)
         await asyncio.sleep(0.15)
-        for attempt in range(5):
+        max_attempts = 20
+        for attempt in range(max_attempts):
             try:
                 self.info = ContraDict(
                     await self._http.get("version"), silent=True
                 )
             except (Exception,):
-                if attempt == 4:
+                if attempt == max_attempts - 1:
                     logger.debug("Could not start", exc_info=True)
                 else:
-                    await self.sleep(0.5)
+                    await self.sleep(0.2)
             else:
                 break
         if not self.info:
@@ -644,9 +654,11 @@ async def start(self=None) -> Browser:
                 %s
                 """ % (dashes, message, dashes)
             )
+        await asyncio.sleep(0.03)
         self.connection = Connection(
             self.info.webSocketDebuggerUrl, browser=self
         )
+        await asyncio.sleep(0.03)
         if self.config.autodiscover_targets:
             logger.debug("Enabling autodiscover targets")
             self.connection.handlers[cdp.target.TargetInfoChanged] = [

seleniumbase/undetected/cdp_driver/cdp_util.py

@@ -115,13 +115,22 @@ def __activate_virtual_display_as_needed(
                             "Starting VDisplay from cdp_util: (%s, %s)"
                             % (_xvfb_width, _xvfb_height)
                         )
-                    _xvfb_display.start()
+                    try:
+                        _xvfb_display.start()
+                    except Exception:
+                        time.sleep(0.03)
+                        _xvfb_display.start()
+                    time.sleep(0.03)
                     if "DISPLAY" not in os.environ.keys():
-                        print(
-                            "\n  X11 display failed! Is Xvfb installed? "
-                            "\n  Try this: `sudo apt install -y xvfb`"
-                        )
-                        __activate_standard_virtual_display()
+                        time.sleep(0.03)
+                        _xvfb_display.start()
+                        time.sleep(0.08)
+                        if "DISPLAY" not in os.environ.keys():
+                            print(
+                                "\n  X11 display failed! Is Xvfb installed? "
+                                "\n  Try this: `sudo apt install -y xvfb`"
+                            )
+                            __activate_standard_virtual_display()
                     else:
                         sb_config._virtual_display = _xvfb_display
                         sb_config.headless_active = True
@@ -781,15 +790,14 @@ async def create_from_driver(driver) -> Browser:
 
 
 def free_port() -> int:
-    """Determines a free port using sockets."""
+    """Find and return a free port number assigned by the OS."""
     import socket
 
-    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    free_socket.bind(("127.0.0.1", 0))
-    free_socket.listen(5)
-    port: int = free_socket.getsockname()[1]
-    free_socket.close()
-    return port
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        # Binding to port 0 lets the OS pick a free port
+        s.bind(("127.0.0.1", 0))
+        s.listen(5)
+        return s.getsockname()[1]
 
 
 def filter_recurse_all(

setup.py

@@ -159,7 +159,7 @@
         'filelock~=3.19.1;python_version<"3.10"',
         'filelock>=3.25.2;python_version>="3.10"',
         'fasteners>=0.20',
-        'mycdp>=1.3.4',
+        'mycdp>=1.3.6',
         'pynose>=1.5.5',
         'platformdirs~=4.4.0;python_version<"3.10"',
         'platformdirs>=4.9.4;python_version>="3.10"',
@@ -177,7 +177,7 @@
         'tabcompleter>=1.4.0',
         'pdbp>=1.8.2',
         'idna>=3.11',
-        'charset-normalizer>=3.4.5,<4',
+        'charset-normalizer>=3.4.6,<4',
         'urllib3>=1.26.20,<2;python_version<"3.10"',
         'urllib3>=1.26.20,<3;python_version>="3.10"',
         'requests~=2.32.5',
@@ -234,7 +234,7 @@
         # Usage: coverage run -m pytest; coverage html; coverage report
         "coverage": [
             'coverage>=7.10.7;python_version<"3.10"',
-            'coverage>=7.13.4;python_version>="3.10"',
+            'coverage>=7.13.5;python_version>="3.10"',
             'pytest-cov>=7.0.0',
         ],
         # pip install -e .[flake8]