PYTHON-4580 Add key_expiration_ms option for DEK cache lifetime (#2186)

Author: ShaneHarvey

doc/changelog.rst

@@ -1,7 +1,23 @@
 Changelog
 =========
 
-Changes in Version 4.11.2 (YYYY/MM/DD)
+Changes in Version 4.12.0 (YYYY/MM/DD)
+--------------------------------------
+
+PyMongo 4.12 brings a number of changes including:
+
+- Support for configuring DEK cache lifetime via the ``key_expiration_ms`` argument to
+  :class:`~pymongo.encryption_options.AutoEncryptionOpts`.
+
+Issues Resolved
+...............
+
+See the `PyMongo 4.12 release notes in JIRA`_ for the list of resolved issues
+in this release.
+
+.. _PyMongo 4.12 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=41916
+
+Changes in Version 4.11.2 (2025/03/05)
 --------------------------------------
 
 Version 4.11.2 is a bug fix release.

pymongo/asynchronous/encryption.py

@@ -445,6 +445,7 @@ def _get_internal_client(
                 bypass_encryption=opts._bypass_auto_encryption,
                 encrypted_fields_map=encrypted_fields_map,
                 bypass_query_analysis=opts._bypass_query_analysis,
+                key_expiration_ms=opts._key_expiration_ms,
             ),
         )
         self._closed = False
@@ -547,11 +548,10 @@ class QueryType(str, enum.Enum):
 
 
 def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
-    opts = MongoCryptOptions(**kwargs)
-    # Opt into range V2 encryption.
-    if hasattr(opts, "enable_range_v2"):
-        opts.enable_range_v2 = True
-    return opts
+    # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
+    if kwargs.get("key_expiration_ms") is None:
+        kwargs.pop("key_expiration_ms", None)
+    return MongoCryptOptions(**kwargs)
 
 
 class AsyncClientEncryption(Generic[_DocumentType]):
@@ -564,6 +564,7 @@ def __init__(
         key_vault_client: AsyncMongoClient[_DocumentTypeArg],
         codec_options: CodecOptions[_DocumentTypeArg],
         kms_tls_options: Optional[Mapping[str, Any]] = None,
+        key_expiration_ms: Optional[int] = None,
     ) -> None:
         """Explicit client-side field level encryption.
 
@@ -630,7 +631,12 @@ def __init__(
             Or to supply a client certificate::
 
               kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}}
+        :param key_expiration_ms: The cache expiration time for data encryption keys.
+            Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
+            Set to 0 to disable key expiration.
 
+        .. versionchanged:: 4.12
+           Added the `key_expiration_ms` parameter.
         .. versionchanged:: 4.0
            Added the `kms_tls_options` parameter and the "kmip" KMS provider.
 
@@ -666,14 +672,19 @@ def __init__(
         key_vault_coll = key_vault_client[db][coll]
 
         opts = AutoEncryptionOpts(
-            kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options
+            kms_providers,
+            key_vault_namespace,
+            kms_tls_options=kms_tls_options,
+            key_expiration_ms=key_expiration_ms,
         )
         self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO(
             None, key_vault_coll, None, opts
         )
         self._encryption = AsyncExplicitEncrypter(
             self._io_callbacks,
-            _create_mongocrypt_options(kms_providers=kms_providers, schema_map=None),
+            _create_mongocrypt_options(
+                kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms
+            ),
         )
         # Use the same key vault collection as the callback.
         assert self._io_callbacks.key_vault_coll is not None
@@ -700,6 +711,7 @@ async def create_encrypted_collection(
         creation. :class:`~pymongo.errors.EncryptionError` will be
         raised if the collection already exists.
 
+        :param database: the database to create the collection
         :param name: the name of the collection to create
         :param encrypted_fields: Document that describes the encrypted fields for
             Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys.  For example:

pymongo/encryption_options.py

@@ -57,6 +57,7 @@ def __init__(
         crypt_shared_lib_required: bool = False,
         bypass_query_analysis: bool = False,
         encrypted_fields_map: Optional[Mapping[str, Any]] = None,
+        key_expiration_ms: Optional[int] = None,
     ) -> None:
         """Options to configure automatic client-side field level encryption.
 
@@ -191,9 +192,14 @@ def __init__(
                       ]
                   }
                 }
+        :param key_expiration_ms: The cache expiration time for data encryption keys.
+            Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
+            Set to 0 to disable key expiration.
 
+        .. versionchanged:: 4.12
+           Added the `key_expiration_ms` parameter.
         .. versionchanged:: 4.2
-           Added `encrypted_fields_map` `crypt_shared_lib_path`, `crypt_shared_lib_required`,
+           Added the `encrypted_fields_map`, `crypt_shared_lib_path`, `crypt_shared_lib_required`,
            and `bypass_query_analysis` parameters.
 
         .. versionchanged:: 4.0
@@ -210,7 +216,6 @@ def __init__(
         if encrypted_fields_map:
             validate_is_mapping("encrypted_fields_map", encrypted_fields_map)
         self._encrypted_fields_map = encrypted_fields_map
-        self._bypass_query_analysis = bypass_query_analysis
         self._crypt_shared_lib_path = crypt_shared_lib_path
         self._crypt_shared_lib_required = crypt_shared_lib_required
         self._kms_providers = kms_providers
@@ -233,6 +238,7 @@ def __init__(
         # Maps KMS provider name to a SSLContext.
         self._kms_ssl_contexts = _parse_kms_tls_options(kms_tls_options)
         self._bypass_query_analysis = bypass_query_analysis
+        self._key_expiration_ms = key_expiration_ms
 
 
 class RangeOpts:

pymongo/synchronous/encryption.py

@@ -442,6 +442,7 @@ def _get_internal_client(
                 bypass_encryption=opts._bypass_auto_encryption,
                 encrypted_fields_map=encrypted_fields_map,
                 bypass_query_analysis=opts._bypass_query_analysis,
+                key_expiration_ms=opts._key_expiration_ms,
             ),
         )
         self._closed = False
@@ -544,11 +545,10 @@ class QueryType(str, enum.Enum):
 
 
 def _create_mongocrypt_options(**kwargs: Any) -> MongoCryptOptions:
-    opts = MongoCryptOptions(**kwargs)
-    # Opt into range V2 encryption.
-    if hasattr(opts, "enable_range_v2"):
-        opts.enable_range_v2 = True
-    return opts
+    # For compat with pymongocrypt <1.13, avoid setting the default key_expiration_ms.
+    if kwargs.get("key_expiration_ms") is None:
+        kwargs.pop("key_expiration_ms", None)
+    return MongoCryptOptions(**kwargs)
 
 
 class ClientEncryption(Generic[_DocumentType]):
@@ -561,6 +561,7 @@ def __init__(
         key_vault_client: MongoClient[_DocumentTypeArg],
         codec_options: CodecOptions[_DocumentTypeArg],
         kms_tls_options: Optional[Mapping[str, Any]] = None,
+        key_expiration_ms: Optional[int] = None,
     ) -> None:
         """Explicit client-side field level encryption.
 
@@ -627,7 +628,12 @@ def __init__(
             Or to supply a client certificate::
 
               kms_tls_options={'kmip': {'tlsCertificateKeyFile': 'client.pem'}}
+        :param key_expiration_ms: The cache expiration time for data encryption keys.
+            Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000.
+            Set to 0 to disable key expiration.
 
+        .. versionchanged:: 4.12
+           Added the `key_expiration_ms` parameter.
         .. versionchanged:: 4.0
            Added the `kms_tls_options` parameter and the "kmip" KMS provider.
 
@@ -659,14 +665,19 @@ def __init__(
         key_vault_coll = key_vault_client[db][coll]
 
         opts = AutoEncryptionOpts(
-            kms_providers, key_vault_namespace, kms_tls_options=kms_tls_options
+            kms_providers,
+            key_vault_namespace,
+            kms_tls_options=kms_tls_options,
+            key_expiration_ms=key_expiration_ms,
         )
         self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO(
             None, key_vault_coll, None, opts
         )
         self._encryption = ExplicitEncrypter(
             self._io_callbacks,
-            _create_mongocrypt_options(kms_providers=kms_providers, schema_map=None),
+            _create_mongocrypt_options(
+                kms_providers=kms_providers, schema_map=None, key_expiration_ms=key_expiration_ms
+            ),
         )
         # Use the same key vault collection as the callback.
         assert self._io_callbacks.key_vault_coll is not None
@@ -693,6 +704,7 @@ def create_encrypted_collection(
         creation. :class:`~pymongo.errors.EncryptionError` will be
         raised if the collection already exists.
 
+        :param database: the database to create the collection
         :param name: the name of the collection to create
         :param encrypted_fields: Document that describes the encrypted fields for
             Queryable Encryption. The "keyId" may be set to ``None`` to auto-generate the data keys.  For example:

test/asynchronous/unified_format.py

@@ -378,6 +378,7 @@ async def drop(self: AsyncGridFSBucket, *args: Any, **kwargs: Any) -> None:
                 opts["key_vault_client"],
                 DEFAULT_CODEC_OPTIONS,
                 opts.get("kms_tls_options", kms_tls_options),
+                opts.get("key_expiration_ms"),
             )
             return
         elif entity_type == "thread":
@@ -439,7 +440,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest):
     a class attribute ``TEST_SPEC``.
     """
 
-    SCHEMA_VERSION = Version.from_string("1.21")
+    SCHEMA_VERSION = Version.from_string("1.22")
     RUN_ON_LOAD_BALANCER = True
     RUN_ON_SERVERLESS = True
     TEST_SPEC: Any

test/asynchronous/utils_spec_runner.py

@@ -18,6 +18,7 @@
 import asyncio
 import functools
 import os
+import time
 import unittest
 from asyncio import iscoroutinefunction
 from collections import abc
@@ -314,6 +315,10 @@ async def assert_index_not_exists(self, database, collection, index):
         coll = self.client[database][collection]
         self.assertNotIn(index, [doc["name"] async for doc in await coll.list_indexes()])
 
+    async def wait(self, ms):
+        """Run the "wait" test operation."""
+        await asyncio.sleep(ms / 1000.0)
+
     def assertErrorLabelsContain(self, exc, expected_labels):
         labels = [l for l in expected_labels if exc.has_error_label(l)]
         self.assertEqual(labels, expected_labels)

test/client-side-encryption/spec/legacy/fle2v2-Rangev2-Compact.json

@@ -6,8 +6,7 @@
         "replicaset",
         "sharded",
         "load-balanced"
-      ],
-      "serverless": "forbid"
+      ]
     }
   ],
   "database_name": "default",