Local sealing improvements (#7554)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Amaury Chamayou <amchamay@microsoft.com>

Author: 3 people

CHANGELOG.md

@@ -48,6 +48,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 - Improved `ccf::historical::verify_self_issued_receipt` - now can verify receipts signed by the past service identities if they were back-endorsed (#7546).
+- Local sealing recovery now stores sealed secrets in the ledger instead of separately on disk. (#7554)
+  - To configure this use `enable_local_sealing` (top-level) and `command.recover.previous_local_sealing_identity` in the configuration.
+  - The configuration options `output_files.sealed_ledger_secret_location` and `command.recover.previous_sealed_ledger_secret_location` have been deprecated and are ignored.
+  - Recovery keys are now stored in `public:ccf.gov.nodes.sealed_recovery_keys` and encrypted shares in `public:ccf.internal.sealed_shares`.
+  - There is an update in the constitution to reseal whenever a node is added, this ensures that as soon as a node is trusted, it can recover from that point in the ledger.
 
 ### Removed
 

CMakeLists.txt

@@ -261,6 +261,7 @@ add_ccf_static_library(
        ${CCF_DIR}/src/node/historical_queries_adapter.cpp
        ${CCF_DIR}/src/node/historical_queries_utils.cpp
        ${CCF_DIR}/src/node/receipt.cpp
+       ${CCF_DIR}/src/node/local_sealing.cpp
   LINK_LIBS http_parser ccfcrypto ccf_kv
 )
 
@@ -675,6 +676,7 @@ if(BUILD_TESTS)
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/historical_queries.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/receipt.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/src/node/receipt.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/local_sealing.cpp
     )
     target_link_libraries(
       historical_queries_test PRIVATE http_parser ccf_kv ccf_endpoints

doc/audit/builtin_maps.rst

@@ -218,6 +218,17 @@ The minimum trusted TCB version for new nodes allowed to join the network (:doc`
    * - ``00a00f11``
      - ``{"hexstring": "d315000000000004", "boot_loader": 4, "tee": 0, "snp": 21, "microcode": 211}``
 
+``nodes.sealed_recovery_keys``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Node ID: SHA-256 digest of the node public key, represented as a hex-encoded string.
+
+**Value** Sealed recovery key for the node. The private key is encrypted using a key derived from SNP's ``DERIVED_KEY``, allowing the node to unseal its recovery share during local sealing recovery.
+
+.. doxygenstruct:: ccf::SealedRecoveryKey
+    :project: CCF
+    :members:
+
 ``service.info``
 ~~~~~~~~~~~~~~~~
 
@@ -559,6 +570,17 @@ While the contents themselves are encrypted, the table is public so as to be acc
 
 **Value** Last signed Merkle root of previous service instance, represented as a hex-encoded string.
 
+``sealed_shares``
+~~~~~~~~~~~~~~~~~
+
+**Value** Per-node encrypted ledger secret wrapping keys, encrypted by the public keys recorded in ``nodes.sealed_recovery_keys``.
+
+While the contents themselves are encrypted, the table is public so as to be accessible by a node starting a recovery service.
+
+.. doxygenstruct:: ccf::SealedSharesInfo
+    :project: CCF
+    :members:
+
 ``last_recovery_type``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 **Value** The mechanism by which the ledger secret was recovered.

doc/host_config_schema/cchost_config.json

@@ -357,9 +357,9 @@
                     "type": "string",
                     "description": "Path to the previous service certificate (PEM) file"
                   },
-                  "previous_sealed_ledger_secret_location": {
-                    "type": ["string"],
-                    "description": "Path to the sealed ledger secret folder, the ledger secrets for the recovered service will be unsealed from here instead of reconstructed from recovery shares."
+                  "previous_local_sealing_identity": {
+                    "type": ["string", "null"],
+                    "description": "The identity of the previous node which sealed the ledger secrets. Required if local sealing is enabled"
                   },
                   "self_healing_open": {
                     "type": "object",
@@ -638,10 +638,6 @@
         "rpc_addresses_file": {
           "type": "string",
           "description": "Path to file in which all RPC addresses (hostnames and ports) will be written to on startup. This option is particularly useful when binding to port 0 and getting auto-assigned a port by the OS. No file is created if this entry is not specified"
-        },
-        "sealed_ledger_secret_location": {
-          "type": "string",
-          "description": "Path to the folder where the node will seal its ledger secrets."
         }
       },
       "description": "This section includes configuration for additional files output by the node",
@@ -713,6 +709,11 @@
       "type": "string",
       "default": "512MB",
       "description": "Historical queries cache soft limit (as size string)"
+    },
+    "enable_local_sealing": {
+      "type": "boolean",
+      "default": false,
+      "description": "Enable sealing of ledger secrets using platform derived key capabilities (e.g. AMD SEV-SNP derived keys). This allows the node to unilaterally recover its ledger secrets on restart without needing to reconstruct them from recovery shares."
     }
   },
   "required": ["network", "command"],

doc/operations/recovery.rst

@@ -116,34 +116,106 @@ Once operators have established a recovered crash-fault tolerant public network,
 Local Sealing Recovery (Experimental)
 -------------------------------------
 
-SNP provides the `DERIVED_KEY` guest message which derives a key from the CPU's VCEK (or VLEK), TCB version and the guest's measurement and host_data (policy), thus any change to the CPU, measurement or policy, or a rolled-back TCB version, will prevent the key from being reconstructed.
-If configured, the node will unseal the secrets it previously sealed instead of waiting for recovery shares from members after `transition_to_open` is triggered.
+SNP provides the ``DERIVED_KEY`` guest message which derives a key from the CPU's VCEK (or VLEK), TCB version and the guest's measurement and host_data (policy), thus any change to the CPU, measurement or policy, or a rolled-back TCB version, will prevent the key from being reconstructed.
+If configured, the node will unseal the secrets it previously sealed instead of waiting for recovery shares from members after ``transition_to_open`` is triggered.
 
-If, in config.json, `output_files.sealed_ledger_secret_location` is set, the node will derive a key and seal versioned ledger secrets to that directory.
-This capability is noted in `public:ccf.gov.node.info[node].will_locally_seal_ledger_secrets`, to allow it to be audited.
+Overview
+~~~~~~~~
 
-Then if `command.recover.previous_sealed_ledger_secret_location` is set in the config.json, when the node recovers and receives the `transition_to_open` transaction, the node will try to unseal the latest ledger secret and use that to recover the ledger.
-If this is unsuccessful, it will fall back to waiting for recovery shares.
-Which of these two paths is taken is noted in the `public:ccf.internal.last_recovery_type`.
+When local sealing is enabled, each node generates an RSA key pair (the "recovery key pair") during join. The private key is encrypted (sealed) using an AES-GCM key derived from the SNP ``DERIVED_KEY``, and the public key along with the sealed private key is stored in the ``public:ccf.gov.nodes.sealed_recovery_keys`` table.
 
-.. code-block:: bash
+During normal operation, whenever the ledger secret changes, or a node joins the network, the system also shuffles "sealed shares". 
+The primary generates a fresh ledger secret wrapping key, encrypts the ledger secret with that key, and stores a sealed copy of the wrapping key for each trusted node with a sealed recovery public key. 
 
-    $ cat /path/to/config/file
-      ...
+During recovery, if the node was previously part of the network and has the same CPU, measurement, and policy, it can bypass the need for member recovery shares by re-deriving the sealing key, unsealing its recovery private key, decrypting the sealed wrapping key, and using that to unwrap the ledger secret. 
+
+The following diagram illustrates the key hierarchy and encryption relationships:
+
+.. mermaid::
+
+    flowchart TB
+        subgraph SNP["SNP PSP"]
+            DK["DERIVED_KEY"]
+            VCEK
+            Measurement
+            Policy["UserData (Policy)"]
+            TCB
+
+            VCEK --> DK
+            Measurement --> DK
+            Policy --> DK            
+            TCB --> DK
+        end
+
+        subgraph KG["Key Generation"]
+            subgraph Sealing Key
+                SK["Sealing Key<br/>(HKDF)"]
+                Label["Label: <br/>CCF AMD Local Sealing Key"]
+                DK -->|ikm| SK
+                Label -->|info| SK
+            end
+            
+            RSA["Recovery Key<br/>(RSA Key Pair)"]
+            PubKey["Public Key"]
+            PrivKey["Private Key"]
+            RSA --> PubKey
+            RSA --> PrivKey
+
+            LS["Ledger secret"]
+            LSWK["Ledger secret wrapping key"]
+        end
+
+        subgraph Sealed["Store: nodes.sealed_recovery_keys"]
+            SPK["Sealed Private Key<br/>(AES-GCM encrypted)"]
+            SK -->|key| SPK
+            PrivKey --> SPK
+
+            StoredPubKey["Public Key (plaintext)"]
+            PubKey --> StoredPubKey
+        end
+
+
+        subgraph Shares["Store: internal.sealed_shares table"]
+            WLS["Wrapped Ledger Secret<br/>(AES-GCM encrypted)"]
+            LSWK -->|key| WLS
+            LS --> WLS
+
+            EWK["Encrypted Wrapping Key<br/>(per-node, RSA-OAEP encrypted)"]
+            StoredPubKey -->|key| EWK
+            LSWK --> EWK
+        end
+
+        subgraph Recovery["Recovery Process"]
+            UPK["Unsealed Private Key"]
+            SK -->|key| UPK
+            SPK --> UPK
+
+            UWK["Unsealed Wrapping Key"]
+            UPK -->|key| UWK
+            EWK --> UWK
+
+            ULS["Unsealed Ledger Secret"]
+            UWK -->|key| ULS
+            WLS --> ULS            
+        end
+
+Configuration
+~~~~~~~~~~~~~
+
+To enable local sealing, set ``enable_local_sealing`` to ``true`` in the node configuration. During recovery, the node's previous identity (node ID) must be specified via ``command.recover.previous_local_sealing_identity`` so the node can look up its sealed share.
+In the future this will be a single shared identifier for both self-healing-open and local sealing recovery, but for now it is simply the previous node ID.
+
+.. code-block:: json
+
+    {
+      "enable_local_sealing": true,
       "command": {
         "type": "Recover",
-        ...
         "recover": {
-          ...
-          "previous_sealed_ledger_secret_location": "/path/to/previous/secret"
+          "previous_local_sealing_identity": "<previous-node-id>"
         }
       }
-      "output_files": {
-        ...
-        "sealed_ledger_secret_location": "/path/to/new/secret"
-      }
-      ...
-    $ /opt/ccf/bin/js_generic --config /path/to/config/file
+    }
 
 Self-Healing-Open recovery (Experimental)
 -----------------------------------------

include/ccf/node/startup_config.h

@@ -4,6 +4,7 @@
 
 #include "ccf/crypto/curve.h"
 #include "ccf/ds/unit_strings.h"
+#include "ccf/entity_id.h"
 #include "ccf/node/cose_signatures_config.h"
 #include "ccf/pal/attestation_sev_snp_endorsements.h"
 #include "ccf/service/consensus_config.h"
@@ -125,7 +126,7 @@ namespace ccf
     std::string service_subject_name = "CN=CCF Service";
     ccf::COSESignaturesConfig cose_signatures;
 
-    std::optional<std::string> sealed_ledger_secret_location;
+    bool enable_local_sealing = false;
 
     nlohmann::json service_data = nullptr;
 
@@ -154,8 +155,7 @@ namespace ccf
     {
       std::optional<std::vector<uint8_t>> previous_service_identity =
         std::nullopt;
-      std::optional<std::string> previous_sealed_ledger_secret_location =
-        std::nullopt;
+      std::optional<NodeId> previous_local_sealing_identity = std::nullopt;
       std::optional<SelfHealingOpenConfig> self_healing_open = std::nullopt;
     };
     Recover recover = {};

include/ccf/service/local_sealing.h

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/crypto/pem.h"
+#include "ccf/ds/json.h"
+#include "ccf/pal/attestation_sev_snp.h"
+
+#include <cstdint>
+
+namespace ccf
+{
+  enum DerivedSealingKeyAlgorithm : uint8_t
+  {
+    SNP_v1 = 0
+  };
+
+  DECLARE_JSON_ENUM(
+    DerivedSealingKeyAlgorithm,
+    {{DerivedSealingKeyAlgorithm::SNP_v1, "SNP_TCB_v1"}});
+
+  struct SealedRecoveryKey
+  {
+    DerivedSealingKeyAlgorithm version = DerivedSealingKeyAlgorithm::SNP_v1;
+    std::vector<uint8_t> ciphertext;
+    crypto::Pem pubkey;
+    pal::snp::TcbVersionRaw tcb_version;
+
+    bool operator==(const SealedRecoveryKey&) const = default;
+  };
+
+  DECLARE_JSON_TYPE(SealedRecoveryKey);
+  DECLARE_JSON_REQUIRED_FIELDS(
+    SealedRecoveryKey, version, ciphertext, pubkey, tcb_version);
+}

include/ccf/service/node_info_network.h

@@ -150,7 +150,6 @@ namespace ccf
     /// RPC interfaces
     RpcInterfaces rpc_interfaces;
 
-    // Denote whether this node will locally seal the ledger secret
     bool will_locally_seal_ledger_secrets = false;
   };
 

samples/constitutions/default/actions.js

@@ -1323,6 +1323,9 @@ const actions = new Map([
             ccf.strToBuf(args.node_id),
             ccf.jsonCompatibleToBuf(nodeInfo),
           );
+          if (ccf.node.shuffleSealedShares !== undefined) {
+            ccf.node.shuffleSealedShares();
+          }
 
           // Also generate and record service-endorsed node certificate from node CSR
           if (nodeInfo.certificate_signing_request !== undefined) {

src/common/configuration.h

@@ -135,9 +135,7 @@ namespace ccf
   DECLARE_JSON_REQUIRED_FIELDS(
     StartupConfig::Recover, previous_service_identity);
   DECLARE_JSON_OPTIONAL_FIELDS(
-    StartupConfig::Recover,
-    previous_sealed_ledger_secret_location,
-    self_healing_open);
+    StartupConfig::Recover, previous_local_sealing_identity, self_healing_open);
 
   DECLARE_JSON_TYPE_WITH_BASE(StartupConfig, CCFConfig);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -152,5 +150,5 @@ namespace ccf
     start,
     join,
     recover,
-    sealed_ledger_secret_location);
+    enable_local_sealing);
 }