MONGOCRYPT-793 support mixing QE and unencrypted JSON schemas (#1076)

Author: connorsmacd

src/mc-schema-broker-private.h

@@ -34,6 +34,8 @@ mc_schema_broker_t *mc_schema_broker_new(void);
 
 void mc_schema_broker_destroy(mc_schema_broker_t *sb);
 
+void mc_schema_broker_support_mixing_schemas(mc_schema_broker_t *sb);
+
 // mc_schema_broker_request requests a schema for a collection. Ignores duplicates.
 // Returns error if two requests have different databases (not-yet supported).
 bool mc_schema_broker_request(mc_schema_broker_t *sb, const char *db, const char *coll, mongocrypt_status_t *status);

src/mc-schema-broker.c

@@ -50,6 +50,7 @@ struct mc_schema_broker_t {
     char *db; // Database shared by all schemas.
     mc_schema_entry_t *ll;
     size_t ll_len;
+    bool schema_mixing_is_supported;
 };
 
 mc_schema_broker_t *mc_schema_broker_new(void) {
@@ -92,6 +93,11 @@ bool mc_schema_broker_has_multiple_requests(const mc_schema_broker_t *sb) {
     return sb->ll_len > 1;
 }
 
+void mc_schema_broker_support_mixing_schemas(mc_schema_broker_t *sb) {
+    BSON_ASSERT_PARAM(sb);
+    sb->schema_mixing_is_supported = true;
+}
+
 void mc_schema_broker_destroy(mc_schema_broker_t *sb) {
     if (!sb) {
         return;
@@ -702,6 +708,11 @@ static bool append_encryptionInformation(const mc_schema_broker_t *sb,
     }
 
     for (mc_schema_entry_t *se = sb->ll; se != NULL; se = se->next) {
+        // encryptedFields is preferred over jsonSchema
+        if (!se->encryptedFields.set && se->jsonSchema.set) {
+            continue;
+        }
+
         BSON_ASSERT(se->satisfied);
         bool loop_ok = false;
         char *ns = bson_strdup_printf("%s.%s", sb->db, se->coll);
@@ -917,10 +928,20 @@ static bool insert_encryptionInformation(const mc_schema_broker_t *sb,
     return ok;
 }
 
+static bool any_entry_includes_encryptedFields(mc_schema_entry_t *head) {
+    for (mc_schema_entry_t *se = head; se != NULL; se = se->next) {
+        if (se->encryptedFields.set) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 // insert_csfleEncryptionSchemas appends schema information to a command for CSFLE.
 // Only consumed by query analysis (mongocryptd/crypt_shared).
 // For one JSON schema, use `jsonSchema` for backwards compatibility.
-// For multiple JSON schemas, use `csfleEncryptionSchemas` (added in server 8.2).
+// For multiple JSON schemas, use `csfleEncryptionSchemas` (added in server 8.1).
 static bool insert_csfleEncryptionSchemas(const mc_schema_broker_t *sb,
                                           bson_t *cmd /* in/out */,
                                           mc_cmd_target_t cmd_target,
@@ -965,7 +986,13 @@ static bool insert_csfleEncryptionSchemas(const mc_schema_broker_t *sb,
         return false;
     }
 
+    const bool skip_empty_schemas = any_entry_includes_encryptedFields(sb->ll);
+
     for (mc_schema_entry_t *se = sb->ll; se != NULL; se = se->next) {
+        if (se->encryptedFields.set || (!se->jsonSchema.set && skip_empty_schemas)) {
+            continue;
+        }
+
         BSON_ASSERT(se->satisfied);
 
         char *ns = bson_strdup_printf("%s.%s", sb->db, se->coll);
@@ -1033,10 +1060,15 @@ bool mc_schema_broker_add_schemas_to_cmd(const mc_schema_broker_t *sb,
     }
 
     if (has_encryptedFields && has_jsonSchema) {
-        // If any collection has encryptedFields, error if any collection only has a JSON Schema.
+        if (sb->schema_mixing_is_supported) {
+            return insert_encryptionInformation(sb, cmd_name, cmd, cmd_target, status)
+                && insert_csfleEncryptionSchemas(sb, cmd, cmd_target, status);
+        }
+
         CLIENT_ERR("Collection '%s' has an encryptedFields configured, but collection '%s' has a JSON schema "
-                   "configured. This is currently not supported. To ignore the JSON schema, add an empty entry for "
-                   "'%s' to AutoEncryptionOpts.encryptedFieldsMap: \"%s\": { \"fields\": [] }",
+                   "configured. This is not supported on mongocryptd/crypt_shared versions below 8.2. To ignore the "
+                   "JSON schema, add an empty entry for '%s' to AutoEncryptionOpts.encryptedFieldsMap: \"%s\": { "
+                   "\"fields\": [] }",
                    coll_with_encryptedFields,
                    coll_with_jsonSchema,
                    coll_with_jsonSchema,

src/mongocrypt-ctx-encrypt.c

@@ -17,6 +17,7 @@
 #include "mc-efc-private.h"
 #include "mc-fle-blob-subtype-private.h"
 #include "mc-fle2-rfds-private.h"
+#include "mc-schema-broker-private.h"
 #include "mc-tokens-private.h"
 #include "mongocrypt-ciphertext-private.h"
 #include "mongocrypt-crypto-private.h"
@@ -2582,9 +2583,11 @@ bool mongocrypt_ctx_encrypt_init(mongocrypt_ctx_t *ctx, const char *db, int32_t
 
 #define WIRE_VERSION_SERVER_6 17
 #define WIRE_VERSION_SERVER_8_1 26
+#define WIRE_VERSION_SERVER_8_2 27
 // The crypt_shared version format is defined in mongo_crypt-v1.h.
 // Example: server 6.2.1 is encoded as 0x0006000200010000
 #define CRYPT_SHARED_8_1 0x0008000100000000ull
+#define CRYPT_SHARED_8_2 0x0008000200000000ull
 
 /* mongocrypt_ctx_encrypt_ismaster_done is called when:
  * 1. The max wire version of mongocryptd is known.
@@ -2626,6 +2629,10 @@ static bool mongocrypt_ctx_encrypt_ismaster_done(mongocrypt_ctx_t *ctx) {
                 _mongocrypt_ctx_fail(ctx);
                 return false;
             }
+
+            if (ectx->ismaster.maxwireversion >= WIRE_VERSION_SERVER_8_2) {
+                mc_schema_broker_support_mixing_schemas(ectx->sb);
+            }
         }
     }
 
@@ -2643,6 +2650,10 @@ static bool mongocrypt_ctx_encrypt_ismaster_done(mongocrypt_ctx_t *ctx) {
                 _mongocrypt_ctx_fail(ctx);
                 return false;
             }
+
+            if (version >= CRYPT_SHARED_8_2) {
+                mc_schema_broker_support_mixing_schemas(ectx->sb);
+            }
         }
     }
 

test/data/lookup/mixed/qe/non-csfle-schema/cmd-to-mongocryptd.json

@@ -0,0 +1,52 @@
+{
+  "aggregate": "c1",
+  "pipeline": [
+    {
+      "$lookup": {
+        "from": "c2",
+        "localField": "joinme",
+        "foreignField": "joinme",
+        "as": "matched"
+      }
+    },
+    {
+      "$match": {
+        "encryptedUnindexed": "foo"
+      }
+    }
+  ],
+  "cursor": {},
+  "encryptionInformation": {
+    "type": {
+      "$numberInt": "1"
+    },
+    "schema": {
+      "db.c1": {
+        "escCollection": "enxcol_.c1.esc",
+        "ecocCollection": "enxcol_.c1.ecoc",
+        "fields": [
+          {
+            "keyId": {
+              "$binary": {
+                "base64": "q83vqxI0mHYSNBI0VniQEg==",
+                "subType": "04"
+              }
+            },
+            "path": "encryptedUnindexed",
+            "bsonType": "string"
+          }
+        ]
+      }
+    }
+  },
+  "csfleEncryptionSchemas": {
+    "db.c2": {
+      "jsonSchema": {
+        "required": [
+          "foo"
+        ]
+      },
+      "isRemoteSchema": true
+    }
+  }
+}

test/data/lookup/mixed/qe/non-csfle-schema/cmd-to-mongod.json

@@ -0,0 +1,47 @@
+{
+  "aggregate": "c1",
+  "pipeline": [
+    {
+      "$lookup": {
+        "from": "c2",
+        "as": "matched",
+        "localField": "joinme",
+        "foreignField": "joinme"
+      }
+    },
+    {
+      "$match": {
+        "encryptedUnindexed": {
+          "$binary": {
+            "base64": "DIkAAAAFZAAgAAAAAJg7KMGBFzQvSG5ipqeBSZ9oz6bVVQ7VWhOlAEb/g286BXMAIAAAAAAyC0j6Jl0oR17xumRgOxHlZSHkHtaQDkaV+ooVn7dVXgVsACAAAAAAxCCKn0pGHRJv9x2o1rDoFdTQfqbGN0anbP9RVNoYbrESY20AAAAAAAAAAAAA",
+            "subType": "06"
+          }
+        }
+      }
+    }
+  ],
+  "cursor": {},
+  "encryptionInformation": {
+    "type": {
+      "$numberInt": "1"
+    },
+    "schema": {
+      "db.c1": {
+        "escCollection": "enxcol_.c1.esc",
+        "ecocCollection": "enxcol_.c1.ecoc",
+        "fields": [
+          {
+            "keyId": {
+              "$binary": {
+                "base64": "q83vqxI0mHYSNBI0VniQEg==",
+                "subType": "04"
+              }
+            },
+            "path": "encryptedUnindexed",
+            "bsonType": "string"
+          }
+        ]
+      }
+    }
+  }
+}

test/data/lookup/mixed/qe/non-csfle-schema/cmd.json

@@ -0,0 +1,19 @@
+{
+  "aggregate": "c1",
+  "pipeline": [
+    {
+      "$lookup": {
+        "from": "c2",
+        "localField": "joinme",
+        "foreignField": "joinme",
+        "as": "matched"
+      }
+    },
+    {
+      "$match": {
+        "encryptedUnindexed": "foo"
+      }
+    }
+  ],
+  "cursor": {}
+}

test/data/lookup/mixed/qe/non-csfle-schema/collInfo-c1.json

@@ -0,0 +1,38 @@
+{
+  "name": "c1",
+  "type": "collection",
+  "options": {
+    "encryptedFields": {
+      "escCollection": "enxcol_.c1.esc",
+      "ecocCollection": "enxcol_.c1.ecoc",
+      "fields": [
+        {
+          "keyId": {
+            "$binary": {
+              "base64": "q83vqxI0mHYSNBI0VniQEg==",
+              "subType": "04"
+            }
+          },
+          "path": "encryptedUnindexed",
+          "bsonType": "string"
+        }
+      ]
+    }
+  },
+  "info": {
+    "readOnly": false,
+    "uuid": {
+      "$binary": {
+        "base64": "1dVWZUeyRZGZ3/NwfKTdLQ==",
+        "subType": "04"
+      }
+    }
+  },
+  "idIndex": {
+    "v": 2,
+    "key": {
+      "_id": 1
+    },
+    "name": "_id_"
+  }
+}

test/data/lookup/mixed/qe/non-csfle-schema/collInfo-c2.json

@@ -0,0 +1,26 @@
+{
+  "type": "collection",
+  "name": "c2",
+  "idIndex": {
+    "ns": "db.c2",
+    "name": "_id_",
+    "key": {
+      "_id": {
+        "$numberInt": "1"
+      }
+    },
+    "v": {
+      "$numberInt": "2"
+    }
+  },
+  "options": {
+    "validator": {
+      "$jsonSchema": {
+        "required": [
+          "foo"
+        ]
+      },
+      "foo": "bar"
+    }
+  }
+}

test/data/lookup/mixed/qe/non-csfle-schema/key-doc.json

@@ -0,0 +1,30 @@
+{
+    "_id": {
+        "$binary": {
+            "base64": "uJ2Njy8YQDuYKbzu2vEKQg==",
+            "subType": "04"
+        }
+    },
+    "keyMaterial": {
+        "$binary": {
+            "base64": "NTpfCae5j0TnbRGsSvHOw7LcIPDhlg8//4N+TQllLZfH0nlj/G3+huCZmWcra+DPH2VbDnmcEmUTCmyPA+Qijjo0v+oL7qNNWUttL4dS8w5GOagQ3/kUtH+ZppgrjbYb1EcPP2G783iYLrTN+9J+fsLV3G36u2hLatGhDmRqDeV9erJOB0bEC69ouki054RWCNJ3AUockMNxxUDe/aQBKw==",
+            "subType": "00"
+        }
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1733927626824"
+        }
+    },
+    "updateDate": {
+        "$date": {
+            "$numberLong": "1733927626824"
+        }
+    },
+    "status": {
+        "$numberInt": "0"
+    },
+    "masterKey": {
+        "provider": "local"
+    }
+}