MNT: Move "backend" from a regular rcParams dict entry to a class attribute

Working towards matplotlib#31791.

> rcParams should be only configuration, not state; i.e. wrt. backend, it
> can hold the desired backend and backend priorities, but it should not
> hold the currently active backend or trigger backend selection as a
> side-effect. The latter causes a lot of complexity in rcParams.

This PR is the first step in the migration strategy outlined there:

> Make the backend a RcParams class attribute and do not hold
> "backend" as a regular dict entry. There can only be one backend,
> so putting that value in a class attribute `RcParams._backend` makes
> sense. This already reduces the complexity as we don't need special
> handling to sync/copy the value when rcParams are bulk-replaced
> (loading from file or rc_context). Instead add special logic to keep
> behavioral backward compatibility with the dict API
> (`rcParams["backend"]` / `rcParams._get("backend")` /
> `"rcParams._set("backend", val)`).

Note on API compatibility: This PR limits itself to the known typical
dict-API usages in the context of backend handling. I've refrained from
trying to fully replicate dict behavior of a "virtual" "backend" parameter,
i.e. `len(rcParams)` / `rcParams.keys()` / `rcParams.values()` /
`rcParams.items()` do not contain "backend" anymore. I would be willing to
accept this level of API breakage. If deemed critical, we could add
compatibility shims for that too.

Author: timhoffm

lib/matplotlib/__init__.py

@@ -693,6 +693,12 @@ class RcParams(MutableMapping, dict):
 
     validate = rcsetup._validators
 
+    # Class-level backend state: shared across all RcParams instances because
+    # there can only be one active backend at a time.  None of the three
+    # RcParams instances (rcParams, rcParamsDefault, rcParamsOrig) stores
+    # "backend" as a dict key; reads/writes are all redirected here.
+    _backend = rcsetup._auto_backend_sentinel
+
     # validate values on the way in
     def __init__(self, *args, **kwargs):
         self.update(*args, **kwargs)
@@ -715,6 +721,9 @@ def _set(self, key, val):
 
         :meta public:
         """
+        if key == "backend":
+            RcParams._backend = val
+            return
         dict.__setitem__(self, key, val)
 
     def _get(self, key):
@@ -736,6 +745,8 @@ def _get(self, key):
 
         :meta public:
         """
+        if key == "backend":
+            return RcParams._backend
         return dict.__getitem__(self, key)
 
     def _update_raw(self, other_params):
@@ -752,25 +763,37 @@ def _update_raw(self, other_params):
             The input mapping from which to update.
         """
         if isinstance(other_params, RcParams):
+            # RcParams never stores "backend" in its dict; skip it here too.
             other_params = dict.items(other_params)
+        else:
+            # Plain dict may contain "backend"; route it through _set so that
+            # it lands in the class variable instead of the instance dict.
+            other_params = list(other_params.items()
+                                if hasattr(other_params, 'items')
+                                else other_params)
+            backend_val = None
+            filtered = []
+            for k, v in other_params:
+                if k == "backend":
+                    backend_val = v
+                else:
+                    filtered.append((k, v))
+            if backend_val is not None:
+                RcParams._backend = backend_val
+            other_params = filtered
         dict.update(self, other_params)
 
-    def _ensure_has_backend(self):
-        """
-        Ensure that a "backend" entry exists.
-
-        Normally, the default matplotlibrc file contains *no* entry for "backend" (the
-        corresponding line starts with ##, not #; we fill in _auto_backend_sentinel
-        in that case.  However, packagers can set a different default backend
-        (resulting in a normal `#backend: foo` line) in which case we should *not*
-        fill in _auto_backend_sentinel.
-        """
-        dict.setdefault(self, "backend", rcsetup._auto_backend_sentinel)
-
     def __setitem__(self, key, val):
-        if (key == "backend"
-                and val is rcsetup._auto_backend_sentinel
-                and "backend" in self):
+        if (key == "backend" and val is rcsetup._auto_backend_sentinel):
+            # Don't let caller silently overwrite a real backend with the auto-sentinel
+            # (only internal code via _set() may do that).
+            #
+            # The primary reason for existence was covering internal rcParams logic
+            # since end-users do not have direct access to
+            # rcsetup._auto_backend_sentinel. It is likely that this is not needed
+            # anymore due to removind "backend" from the dict and  making it
+            # a class attribute RcParams._backend`. But to be on the safe side, we
+            # keep this as long as "backend" is a valid key for rcParams.
             return
         valid_key = _api.getitem_checked(
             self.validate, rcParam=key, _error_cls=KeyError
@@ -784,18 +807,24 @@ def __setitem__(self, key, val):
         self._set(key, cval)
 
     def __getitem__(self, key):
-        # In theory, this should only ever be used after the global rcParams
-        # has been set up, but better be safe e.g. in presence of breakpoints.
-        if key == "backend" and self is globals().get("rcParams"):
-            val = self._get(key)
-            if val is rcsetup._auto_backend_sentinel:
+        if key == "backend":
+            # In theory, this should only ever be used after the global rcParams
+            # has been set up, but better be safe e.g. in presence of breakpoints.
+            if (self is globals().get("rcParams")
+                    and RcParams._backend is rcsetup._auto_backend_sentinel):
                 from matplotlib import pyplot as plt
                 plt.switch_backend(rcsetup._auto_backend_sentinel)
-        return self._get(key)
+            return RcParams._backend
+        return dict.__getitem__(self, key)
+
+    def __contains__(self, key):
+        if key == "backend":
+            return True
+        return dict.__contains__(self, key)
 
     def _get_backend_or_none(self):
         """Get the requested backend, if any, without triggering resolution."""
-        backend = self._get("backend")
+        backend = RcParams._backend
         return None if backend is rcsetup._auto_backend_sentinel else backend
 
     def __repr__(self):
@@ -992,7 +1021,6 @@ def rc_params_from_file(fname, fail_on_error=False, use_default_template=True):
     transform=lambda line: line[1:] if line.startswith("#") else line,
     fail_on_error=True)
 rcParamsDefault._update_raw(rcsetup._hardcoded_defaults)
-rcParamsDefault._ensure_has_backend()
 
 rcParams = RcParams()  # The global instance.
 rcParams._update_raw(rcParamsDefault)
@@ -1201,8 +1229,7 @@ def rc_context(rc=None, fname=None):
             plt.plot(x, y)
 
     """
-    orig = dict(rcParams.copy())
-    del orig['backend']
+    orig = rcParams.copy()
     try:
         if fname:
             rc_file(fname)

lib/matplotlib/__init__.pyi

@@ -78,7 +78,6 @@ class RcParams(dict[RcKeyType, Any]):
 
     def _update_raw(self, other_params: dict | RcParams) -> None: ...
 
-    def _ensure_has_backend(self) -> None: ...
     def __setitem__(self, key: RcKeyType, val: Any) -> None: ...
     def __getitem__(self, key: RcKeyType) -> Any: ...
     def __iter__(self) -> Generator[RcKeyType, None, None]: ...

lib/matplotlib/tests/test_rcparams.py

@@ -686,7 +686,13 @@ def test_rc_aliases(group, option, alias, value):
 
 
 def test_all_params_defined_as_code():
-    assert set(p.name for p in rcsetup._params_list()) == set(mpl.rcParams.keys())
+    params_in_code = {p.name for p in rcsetup._params_list()}
+    # 'backend' is stored in RcParams._backend (a class variable) rather than
+    # in the underlying dict, so it does not appear in rcParams.keys() /
+    # __iter__.  It is still accessible via rcParams['backend'] and
+    # 'backend' in rcParams; it just doesn't show up during iteration.
+    params_in_code.remove("backend")
+    assert params_in_code == set(mpl.rcParams.keys())
 
 
 def test_validators_defined_as_code():

lib/matplotlib/tests/test_typing.py

@@ -104,6 +104,15 @@ def test_rcparam_stubs():
         if not name.startswith('_')
     }
 
+    # backend is not a regular dict key anymore, but we have special logic to ensure
+    # read and write access to it.
+    # The _get('backend') is just a smoke test that 'backend' is still supported, and
+    # it is thus justified to add 'backend' to the keys. This will fail in the future
+    # when we remove 'backend' as accepted key and will remind us that we have to
+    # remove it from the stubs.
+    plt.rcParamsDefault._get('backend')
+    runtime_rc_keys.add('backend')
+
     assert {*typing.get_args(RcKeyType)} == runtime_rc_keys
 
     runtime_rc_group_keys = set()