Selenium 4 support (#166)

Make tor-browser-selenium compatible with Selenium 4.

The following TorBrowserDriver init parameters are changed:

- `timeout` (removed):  not supported by webdriver
- `options` (selenium.webdriver.firefox.options.Options, added)
- `use_custom_profile` (Bool, added)


- Add tests to make sure the builtin addons (NoScript and HTTPE) come installed.
- Add tests to check whether we can install custom extensions.
- Remove the profile directory only if it's temporary (use_custom_profile=False).
- Remove the missing browser test
- Fix the platform name (linux2 -> linux)
* Use the correct profile path
* Prevent exceptions due to undefined last_err.
* Pick a random Marionette port when custom profile is not used.

Hardcoding the Marionette port (2828) makes it impossible to use tbselenium in parallel. After the following bug is fixed
we can stop passing the hardcoded port all together: https://bugzilla.mozilla.org/show_bug.cgi?id=1421766
* Use geckodriver 30 in the CI tests

Author: gunesacar

tbselenium/common.py

@@ -14,6 +14,10 @@
 DEFAULT_TOR_DATA_PATH = join(DEFAULT_TBB_DATA_DIR, 'Tor')
 TB_CHANGE_LOG_PATH = join(DEFAULT_TBB_TORBROWSER_DIR,
                           'Docs', 'ChangeLog.txt')
+# noscript .xpi path as found in the TBB distributions
+DEFAULT_TBB_NO_SCRIPT_XPI_PATH = join(
+    DEFAULT_TBB_PROFILE_PATH, 'extensions',
+    '{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi')
 
 # Directories for bundled fonts - Linux only
 DEFAULT_FONTCONFIG_PATH = join(DEFAULT_TBB_DATA_DIR, 'fontconfig')
@@ -37,7 +41,6 @@
 PORT_BAN_PREFS = ["extensions.torbutton.banned_ports",
                   "network.security.ports.banned"]
 
-TB_INIT_TIMEOUT = 60
 
 # Test constants
 CHECK_TPO_URL = "http://check.torproject.org"

tbselenium/tbdriver.py

@@ -2,23 +2,20 @@
 from os import environ, chdir
 from os.path import isdir, isfile, join, abspath
 from time import sleep
-from selenium import webdriver
+from http.client import CannotSendRequest
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
-from selenium.common.exceptions import WebDriverException
+from selenium.webdriver.firefox.service import Service
 from selenium.webdriver.firefox.webdriver import WebDriver as FirefoxDriver
 from selenium.webdriver.firefox.options import Options
+from selenium.common.exceptions import WebDriverException
 import tbselenium.common as cm
 from tbselenium.utils import prepend_to_env_var, is_busy
 from tbselenium.tbbinary import TBBinary
-from tbselenium.exceptions import (TBDriverConfigError, TBDriverPortError,
-                                   TBDriverPathError)
+from tbselenium.exceptions import (
+    TBDriverConfigError, TBDriverPortError, TBDriverPathError)
 
-try:
-    from httplib import CannotSendRequest
-except ImportError:
-    from http.client import CannotSendRequest
 
 DEFAULT_BANNED_PORTS = "9050,9051,9150,9151"
 
@@ -38,42 +35,83 @@ def __init__(self,
                  pref_dict={},
                  socks_port=None,
                  control_port=None,
-                 extensions=None,
+                 extensions=[],
                  default_bridge_type="",
-                 capabilities=None,
-                 headless=False):
-
+                 headless=False,
+                 options=None,
+                 use_custom_profile=False
+                 ):
+
+        # use_custom_profile: whether to launch from and *write to* the given
+        # profile
+        # False: copy the profile to a tempdir; remove the temp folder on quit
+        # True: use the given profile without copying. This can be used to keep
+        # a stateful profile across different launches of the Tor Browser.
+        # It uses firefox's `-profile`` command line parameter under the hood
+
+        self.use_custom_profile = use_custom_profile
         self.tor_cfg = tor_cfg
+
         self.setup_tbb_paths(tbb_path, tbb_fx_binary_path,
                              tbb_profile_path, tor_data_dir)
-        self.profile = webdriver.FirefoxProfile(self.tbb_profile_path)
-        self.install_extensions(extensions)
+        self.options = Options() if options is None else options
+        install_noscript = False
+
+        USE_DEPRECATED_PROFILE_METHOD = True
+        if self.use_custom_profile:
+            # launch from and write to this custom profile
+            self.options.add_argument("-profile")
+            self.options.add_argument(self.tbb_profile_path)
+        elif USE_DEPRECATED_PROFILE_METHOD:
+            # launch from this custom profile
+            self.options.profile = self.tbb_profile_path
+        else:
+            # Launch with no profile at all. This should be used with caution.
+            # NoScript does not come installed on browsers launched by this
+            # method, so we install it ourselves
+            install_noscript = True
+
         self.init_ports(tor_cfg, socks_port, control_port)
         self.init_prefs(pref_dict, default_bridge_type)
-        self.setup_capabilities(capabilities)
         self.export_env_vars()
-        self.binary = self.get_tb_binary(logfile=tbb_logfile_path)
-        self.binary.add_command_line_options('--class', '"Tor Browser"')
-        options = Options()
+        # TODO:
+        # self.binary = self.get_tb_binary(logfile=tbb_logfile_path)
+        if use_custom_profile:
+            tbb_service = Service(
+                executable_path=executable_path,
+                log_path=tbb_logfile_path,
+                service_args=["--marionette-port", "2828"]
+                )
+        else:
+            tbb_service = Service(
+                executable_path=executable_path,
+                log_path=tbb_logfile_path
+                )
+        self.options.binary = self.tbb_fx_binary_path
+        self.options.add_argument('--class')
+        self.options.add_argument('"Tor Browser"')
         if headless:
-            options.set_headless()
+            self.options.headless = True
+
         super(TorBrowserDriver, self).__init__(
-            firefox_profile=self.profile,
-            firefox_binary=self.binary,
-            capabilities=self.capabilities,
-            timeout=cm.TB_INIT_TIMEOUT,
-            service_log_path=tbb_logfile_path,
+            service=tbb_service,
             executable_path=executable_path,
-            options=options)
+            options=self.options,
+            )
         self.is_running = True
+        self.install_extensions(extensions, install_noscript)
+        self.temp_profile_dir = self.capabilities["moz:profile"]
         sleep(1)
 
-    def install_extensions(self, extensions):
-        """Install the given extension to the profile we are launching."""
-        if extensions is None:
-            return
+    def install_extensions(self, extensions, install_noscript):
+        """Install the given extensions to the profile we are launching."""
+        if install_noscript:
+            no_script_xpi = join(
+                self.tbb_path, cm.DEFAULT_TBB_NO_SCRIPT_XPI_PATH)
+            extensions.append(no_script_xpi)
+
         for extension in extensions:
-            self.profile.add_extension(extension)
+            self.install_addon(extension)
 
     def init_ports(self, tor_cfg, socks_port, control_port):
         """Check SOCKS port and Tor config inputs."""
@@ -174,8 +212,8 @@ def add_ports_to_fx_banned_ports(self, socks_port, control_port):
         """
         if socks_port in cm.KNOWN_SOCKS_PORTS:
             return
-        tb_prefs = self.profile.default_preferences
-        set_pref = self.profile.set_preference
+        tb_prefs = self.options.preferences
+        set_pref = self.options.set_preference
 
         for port_ban_pref in cm.PORT_BAN_PREFS:
             banned_ports = tb_prefs.get(port_ban_pref, DEFAULT_BANNED_PORTS)
@@ -188,7 +226,7 @@ def set_tb_prefs_for_using_system_tor(self, control_port):
 
         We set these prefs for running with Tor started with Stem as well.
         """
-        set_pref = self.profile.set_preference
+        set_pref = self.options.set_preference
         # Prevent Tor Browser running its own Tor process
         set_pref('extensions.torlauncher.start_tor', False)
         # TODO: investigate whether these prefs are up to date or not
@@ -206,11 +244,15 @@ def set_tb_prefs_for_using_system_tor(self, control_port):
         set_pref('extensions.torlauncher.loglevel', 2)
         set_pref('extensions.torlauncher.logmethod', 0)
         set_pref('extensions.torlauncher.prompt_at_startup', False)
+        # disable XPI signature checking
+        set_pref('xpinstall.signatures.required', False)
+        set_pref('xpinstall.whitelist.required', False)
 
     def init_prefs(self, pref_dict, default_bridge_type):
         self.add_ports_to_fx_banned_ports(self.socks_port, self.control_port)
-        set_pref = self.profile.set_preference
+        set_pref = self.options.set_preference
         set_pref('browser.startup.page', "0")
+        set_pref('torbrowser.settings.quickstart.enabled', True)
         set_pref('browser.startup.homepage', 'about:newtab')
         set_pref('extensions.torlauncher.prompt_at_startup', 0)
         # load strategy normal is equivalent to "onload"
@@ -233,7 +275,7 @@ def init_prefs(self, pref_dict, default_bridge_type):
         # pref_dict overwrites above preferences
         for pref_name, pref_val in pref_dict.items():
             set_pref(pref_name, pref_val)
-        self.profile.update_preferences()
+        # self.profile.update_preferences()
 
     def export_env_vars(self):
         """Setup LD_LIBRARY_PATH and HOME environment variables.
@@ -249,22 +291,6 @@ def export_env_vars(self):
         # Add "TBB_DIR/Browser" to the PATH, see issue #10.
         prepend_to_env_var("PATH", self.tbb_browser_dir)
 
-    def setup_capabilities(self, caps):
-        """Setup the required webdriver capabilities."""
-        if caps is None:
-            self.capabilities = {
-                "marionette": True,
-                "capabilities": {
-                    "alwaysMatch": {
-                        "moz:firefoxOptions": {
-                            "log": {"level": "info"}
-                        }
-                    }
-                }
-            }
-        else:
-            self.capabilities = caps
-
     def get_tb_binary(self, logfile=None):
         """Return FirefoxBinary pointing to the TBB's firefox binary."""
         tbb_logfile = open(logfile, 'a+') if logfile else None
@@ -280,13 +306,13 @@ def clean_up_profile_dirs(self):
         """Remove temporary profile directories.
         Only called when WebDriver.quit() is interrupted
         """
-        tempfolder = self.profile.tempfolder
-        profile_path = self.profile.path
+        if self.use_custom_profile:
+            # don't remove the profile if we are writing into it
+            # i.e. stateful mode
+            return
 
-        if tempfolder and isdir(tempfolder):
-            shutil.rmtree(tempfolder)
-        if isdir(profile_path):
-            shutil.rmtree(profile_path)
+        if self.temp_profile_dir and isdir(self.temp_profile_dir):
+            shutil.rmtree(self.temp_profile_dir)
 
     def quit(self):
         """Quit the driver. Clean up if the parent's quit fails."""
@@ -295,11 +321,10 @@ def quit(self):
             super(TorBrowserDriver, self).quit()
         except (CannotSendRequest, AttributeError, WebDriverException):
             try:  # Clean up  if webdriver.quit() throws
-                if self.w3c:
+                if hasattr(self, "service"):
                     self.service.stop()
-                if hasattr(self, "binary"):
-                    self.binary.kill()
-                if hasattr(self, "profile"):
+                if hasattr(self, "options") and hasattr(
+                        self.options, "profile"):
                     self.clean_up_profile_dirs()
             except Exception as e:
                 print("[tbselenium] Exception while quitting: %s" % e)

tbselenium/test/conftest.py

@@ -10,6 +10,7 @@
 
 
 def launch_tor():
+    # TODO: Consider using system tor (when available) to speed tests up
     tor_process = None
     temp_data_dir = tempfile.mkdtemp()
     torrc = {'ControlPort': str(cm.STEM_CONTROL_PORT),

tbselenium/test/fixtures.py

@@ -7,10 +7,7 @@
 from tbselenium.utils import launch_tbb_tor_with_stem, is_busy, read_file
 from selenium.common.exceptions import TimeoutException, WebDriverException
 
-try:
-    from httplib import CannotSendRequest
-except ImportError:
-    from http.client import CannotSendRequest
+from http.client import CannotSendRequest
 
 
 MAX_FIXTURE_TRIES = 3
@@ -31,18 +28,19 @@ def __init__(self, *args, **kwargs):
                 return super(TBDriverFixture, self).__init__(*args, **kwargs)
             except (TimeoutException, WebDriverException,
                     socket_error) as last_err:
-                print ("\nTBDriver init error. Attempt %s %s" %
-                       ((tries + 1), last_err))
+                print("\nERROR: TBDriver init error. Attempt %s %s" %
+                      ((tries + 1), last_err))
                 if FORCE_TB_LOGS_DURING_TESTS:
                     logs = read_file(log_file)
                     if len(logs):
                         print("TB logs:\n%s\n(End of TB logs)" % logs)
                 super(TBDriverFixture, self).quit()  # clean up
                 continue
         # Raise if we didn't return yet
-        to_raise = last_err if last_err else\
-            TorBrowserDriverInitError("Cannot initialize")
-        raise to_raise
+        try:
+            raise last_err
+        except Exception:
+            raise TorBrowserDriverInitError("Cannot initialize")
 
     def __del__(self):
         # remove the temp log file if we created
@@ -83,13 +81,14 @@ def load_url_ensure(self, *args, **kwargs):
                     return
             except (TimeoutException,
                     CannotSendRequest) as last_err:
-                print ("\nload_url timed out.  Attempt %s %s" %
-                       ((tries + 1), last_err))
+                print("\nload_url timed out.  Attempt %s %s" %
+                      ((tries + 1), last_err))
                 continue
         # Raise if we didn't return yet
-        to_raise = last_err if last_err else\
-            WebDriverException("Can't load the page")
-        raise to_raise
+        try:
+            raise last_err
+        except Exception:
+            raise WebDriverException("Can't load the page")
 
 
 def launch_tbb_tor_with_stem_fixture(*args, **kwargs):
@@ -98,12 +97,13 @@ def launch_tbb_tor_with_stem_fixture(*args, **kwargs):
         try:
             return launch_tbb_tor_with_stem(*args, **kwargs)
         except OSError as last_err:
-            print ("\nlaunch_tor try %s %s" % ((tries + 1), last_err))
+            print("\nlaunch_tor try %s %s" % ((tries + 1), last_err))
             if "timeout without success" in str(last_err):
                 continue
             else:  # we don't want to retry if this is not a timeout
                 raise
     # Raise if we didn't return yet
-    to_raise = last_err if last_err else\
-        StemLaunchError("Cannot start Tor")
-    raise to_raise
+    try:
+        raise last_err
+    except Exception:
+        raise StemLaunchError("Cannot start Tor")