Update CDP Mode

Author: mdmintz

seleniumbase/core/sb_cdp.py

@@ -391,7 +391,7 @@ def select(self, selector, timeout=None):
 
     def select_all(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         self.__add_light_pause()
         selector = self.__convert_to_css_if_xpath(selector)
         if not self.is_element_present(selector):
@@ -414,12 +414,12 @@ def select_all(self, selector, timeout=None):
 
     def find_elements(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         return self.select_all(selector, timeout=timeout)
 
     def find_visible_elements(self, selector, timeout=None):
         if not timeout:
-            timeout = settings.SMALL_TIMEOUT
+            timeout = settings.MINI_TIMEOUT
         visible_elements = []
         elements = self.select_all(selector, timeout=timeout)
         for element in elements:

seleniumbase/undetected/cdp_driver/browser.py

@@ -39,22 +39,30 @@ def deconstruct_browser():
     for _ in __registered__instances__:
         if not _.stopped:
             _.stop(deconstruct=True)
-        for attempt in range(5):
+        max_attempts = 5
+        for attempt in range(max_attempts):
             try:
                 if _.config and not _.config.uses_custom_data_dir:
-                    shutil.rmtree(_.config.user_data_dir, ignore_errors=False)
+                    if os.path.exists(_.config.user_data_dir):
+                        shutil.rmtree(
+                            _.config.user_data_dir, ignore_errors=False
+                        )
+                    if not os.path.exists(_.config.user_data_dir):
+                        break
+                    else:
+                        time.sleep(0.12)
             except FileNotFoundError:
                 break
             except (PermissionError, OSError) as e:
-                if attempt == 4:
+                if attempt == max_attempts - 1:
                     logger.debug(
                         "Problem removing data dir %s\n"
                         "Consider checking whether it's there "
                         "and remove it by hand\nerror: %s"
                         % (_.config.user_data_dir, e)
                     )
                     break
-                time.sleep(0.15)
+                time.sleep(0.12)
                 continue
         logging.debug("Temp profile %s was removed." % _.config.user_data_dir)
 
@@ -207,7 +215,8 @@ async def _handle_target_update(
             target_info = event.target_info
             current_tab = next(
                 filter(
-                    lambda item: item.target_id == target_info.target_id, self.targets  # noqa
+                    lambda item: item.target_id == target_info.target_id,
+                    self.targets,
                 )
             )
             current_target = current_tab.target
@@ -583,11 +592,11 @@ async def start(self=None) -> Browser:
                         else "c:/path/to/your/browser.exe"
                     )
                 )
-        if getattr(self.config, "_extensions", None):  # noqa
+        if getattr(self.config, "_extensions", None):
             self.config.add_argument(
                 "--load-extension=%s"
                 % ",".join(str(_) for _ in self.config._extensions)
-            )  # noqa
+            )
         exe = self.config.browser_executable_path
         params = self.config()
         logger.debug(
@@ -613,16 +622,17 @@ async def start(self=None) -> Browser:
         await asyncio.sleep(0.05)
         get_registered_instances().add(self)
         await asyncio.sleep(0.15)
-        for attempt in range(5):
+        max_attempts = 20
+        for attempt in range(max_attempts):
             try:
                 self.info = ContraDict(
                     await self._http.get("version"), silent=True
                 )
             except (Exception,):
-                if attempt == 4:
+                if attempt == max_attempts - 1:
                     logger.debug("Could not start", exc_info=True)
                 else:
-                    await self.sleep(0.5)
+                    await self.sleep(0.2)
             else:
                 break
         if not self.info:
@@ -644,9 +654,11 @@ async def start(self=None) -> Browser:
                 %s
                 """ % (dashes, message, dashes)
             )
+        await asyncio.sleep(0.03)
         self.connection = Connection(
             self.info.webSocketDebuggerUrl, browser=self
         )
+        await asyncio.sleep(0.03)
         if self.config.autodiscover_targets:
             logger.debug("Enabling autodiscover targets")
             self.connection.handlers[cdp.target.TargetInfoChanged] = [

seleniumbase/undetected/cdp_driver/cdp_util.py

@@ -115,13 +115,22 @@ def __activate_virtual_display_as_needed(
                             "Starting VDisplay from cdp_util: (%s, %s)"
                             % (_xvfb_width, _xvfb_height)
                         )
-                    _xvfb_display.start()
+                    try:
+                        _xvfb_display.start()
+                    except Exception:
+                        time.sleep(0.03)
+                        _xvfb_display.start()
+                    time.sleep(0.03)
                     if "DISPLAY" not in os.environ.keys():
-                        print(
-                            "\n  X11 display failed! Is Xvfb installed? "
-                            "\n  Try this: `sudo apt install -y xvfb`"
-                        )
-                        __activate_standard_virtual_display()
+                        time.sleep(0.03)
+                        _xvfb_display.start()
+                        time.sleep(0.08)
+                        if "DISPLAY" not in os.environ.keys():
+                            print(
+                                "\n  X11 display failed! Is Xvfb installed? "
+                                "\n  Try this: `sudo apt install -y xvfb`"
+                            )
+                            __activate_standard_virtual_display()
                     else:
                         sb_config._virtual_display = _xvfb_display
                         sb_config.headless_active = True
@@ -781,15 +790,14 @@ async def create_from_driver(driver) -> Browser:
 
 
 def free_port() -> int:
-    """Determines a free port using sockets."""
+    """Find and return a free port number assigned by the OS."""
     import socket
 
-    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    free_socket.bind(("127.0.0.1", 0))
-    free_socket.listen(5)
-    port: int = free_socket.getsockname()[1]
-    free_socket.close()
-    return port
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        # Binding to port 0 lets the OS pick a free port
+        s.bind(("127.0.0.1", 0))
+        s.listen(5)
+        return s.getsockname()[1]
 
 
 def filter_recurse_all(