Backport #7755 (#7757)

achamayou · maxtropets · web-flow · commit 8d3e6d55c84f · 2026-03-23T19:27:22.000Z
Co-authored-by: Max &lt;maxtropets@microsoft.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.26]
+
+[6.0.26]: https://github.com/microsoft/CCF/releases/tag/ccf-6.0.26
+
+### Fixed
+
+- Fixed cache size calculations for historical queries, resolving a bug where signature transactions could become orphaned and fill the cache's useful space, resulting in incoming user-requested stores being immediately evicted (#7755).
+
 ## [6.0.25]
 
 [6.0.25]: https://github.com/microsoft/CCF/releases/tag/ccf-6.0.25
diff --git a/doc/schemas/node_openapi.json b/doc/schemas/node_openapi.json
@@ -237,6 +237,17 @@
         ],
         "type": "object"
       },
+      "GetHistoricalCacheInfo__Out": {
+        "properties": {
+          "estimated_size": {
+            "$ref": "#/components/schemas/uint64"
+          }
+        },
+        "required": [
+          "estimated_size"
+        ],
+        "type": "object"
+      },
       "GetNetworkInfo__Out": {
         "properties": {
           "current_service_create_txid": {
@@ -921,7 +932,7 @@
   "info": {
     "description": "This API provides public, uncredentialed access to service and node state.",
     "title": "CCF Public Node API",
-    "version": "4.16.0"
+    "version": "4.17.0"
   },
   "openapi": "3.0.0",
   "paths": {
@@ -1100,6 +1111,29 @@
         }
       }
     },
+    "/node/historical_cache": {
+      "get": {
+        "operationId": "GetNodeHistoricalCache",
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/GetHistoricalCacheInfo__Out"
+                }
+              }
+            },
+            "description": "Default response description"
+          },
+          "default": {
+            "$ref": "#/components/responses/default"
+          }
+        },
+        "x-ccf-forwarding": {
+          "$ref": "#/components/x-ccf-forwarding/sometimes"
+        }
+      }
+    },
     "/node/index/strategies": {
       "get": {
         "operationId": "GetNodeIndexStrategies",
diff --git a/include/ccf/historical_queries_interface.h b/include/ccf/historical_queries_interface.h
@@ -196,5 +196,9 @@ namespace ccf::historical
      * more aggressively than waiting for the states to expire.
      */
     virtual bool drop_cached_states(RequestHandle handle) = 0;
+
+    /** Get the estimated size in bytes of all cached stores.
+     */
+    virtual size_t get_estimated_store_cache_size() = 0;
   };
 }
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "6.0.25"
+version = "6.0.26"
 authors = [
   { name="CCF Team", email="CCF-Sec@microsoft.com" },
 ]
diff --git a/src/node/historical_queries.h b/src/node/historical_queries.h
@@ -21,6 +21,7 @@
 
 #ifdef ENABLE_HISTORICAL_VERBOSE_LOGGING
 #  define HISTORICAL_LOG(...) LOG_INFO_FMT(__VA_ARGS__)
+#  include <ranges>
 #else
 #  define HISTORICAL_LOG(...)
 #endif
@@ -64,7 +65,6 @@ FMT_END_NAMESPACE
 namespace ccf::historical
 {
   static constexpr auto slow_fetch_threshold = std::chrono::milliseconds(1000);
-  static constexpr size_t soft_to_raw_ratio{5};
 
   static std::optional<ccf::PrimarySignature> get_signature(
     const ccf::kv::StorePtr& sig_store)
@@ -273,9 +273,17 @@ namespace ccf::historical
             else if (prev_it != my_stores.end() && *new_it > prev_it->first)
             {
               // No longer looking for a seqno which was previously requested.
-              // Remove it from my_stores
-              removed.push_back(prev_it->first);
-              prev_it = my_stores.erase(prev_it);
+              if (
+                supporting_signatures.find(prev_it->first) ==
+                supporting_signatures.end())
+              {
+                removed.push_back(prev_it->first);
+                prev_it = my_stores.erase(prev_it);
+              }
+              else
+              {
+                ++prev_it;
+              }
             }
             else
             {
@@ -311,15 +319,31 @@ namespace ccf::historical
           if (prev_it != my_stores.end())
           {
             // If we have a suffix of seqnos previously requested, now
-            // unrequested, purge them
-            for (auto it = prev_it; it != my_stores.end(); ++it)
+            // unrequested, purge them - but keep supporting signature entries
+            auto it = prev_it;
+            while (it != my_stores.end())
             {
-              removed.push_back(it->first);
+              if (
+                supporting_signatures.find(it->first) ==
+                supporting_signatures.end())
+              {
+                removed.push_back(it->first);
+                it = my_stores.erase(it);
+              }
+              else
+              {
+                ++it;
+              }
             }
-            my_stores.erase(prev_it, my_stores.end());
           }
         }
 
+        HISTORICAL_LOG(
+          "Added seqnos: {}, removed seqnos: {}, supporting signatures: {}",
+          fmt::join(added, ","),
+          fmt::join(removed, ","),
+          fmt::join(std::views::keys(supporting_signatures), ","));
+
         const bool any_diff = !removed.empty() || !added.empty();
 
         if (!any_diff && (should_include_receipts == include_receipts))
@@ -342,13 +366,36 @@ namespace ccf::historical
 
           for (auto seqno : new_seqnos)
           {
-            populate_receipts(seqno);
+            auto more_to_add = populate_receipts(seqno);
+            std::sort(added.begin(), added.end());
+            std::sort(more_to_add.begin(), more_to_add.end());
+
+            std::vector<SeqNo> together;
+            std::merge(
+              added.begin(),
+              added.end(),
+              more_to_add.begin(),
+              more_to_add.end(),
+              std::back_inserter(together));
+
+            if (more_to_add.size() + added.size() != together.size())
+            {
+              LOG_FAIL_FMT(
+                "Invariant violation in adjust_ranges: more_to_add({}) + "
+                "added({}) != together({})",
+                more_to_add.size(),
+                added.size(),
+                together.size());
+              assert(false);
+            }
+
+            std::swap(added, together);
           }
         }
         return {removed, added};
       }
 
-      void populate_receipts(ccf::SeqNo new_seqno)
+      std::vector<ccf::SeqNo> populate_receipts(ccf::SeqNo new_seqno)
       {
         HISTORICAL_LOG(
           "Looking at {}, and populating receipts from it", new_seqno);
@@ -376,6 +423,16 @@ namespace ccf::historical
             HISTORICAL_LOG("{} is not a signature", new_seqno);
             supporting_signatures.erase(new_seqno);
 
+            if (new_details->receipt != nullptr)
+            {
+              HISTORICAL_LOG(
+                "Already have a receipt for {}, so no need to populate more",
+                new_seqno);
+              return {};
+            }
+
+            std::vector<SeqNo> added;
+
             auto next_seqno = new_seqno + 1;
             while (true)
             {
@@ -387,6 +444,18 @@ namespace ccf::historical
                 HISTORICAL_LOG(
                   "Looking for new supporting signature at {}", next_seqno);
                 details = std::make_shared<StoreDetails>();
+                auto my_it = my_stores.find(next_seqno);
+                if (my_it == my_stores.end())
+                {
+                  LOG_TRACE_FMT(
+                    "Tracking potential supporting signature for new seqno {} "
+                    "at {}",
+                    new_seqno,
+                    next_seqno);
+                  added.push_back(next_seqno);
+                  my_stores.insert_or_assign(my_it, next_seqno, details);
+                }
+
                 all_stores.insert_or_assign(all_it, next_seqno, details);
               }
 
@@ -400,7 +469,7 @@ namespace ccf::historical
                   next_seqno,
                   new_seqno);
                 supporting_signatures[next_seqno] = details;
-                return;
+                return added;
               }
               else if (details->is_signature)
               {
@@ -418,7 +487,7 @@ namespace ccf::historical
                     new_seqno));
                 }
 
-                return;
+                return added;
               }
               else
               {
@@ -427,8 +496,11 @@ namespace ccf::historical
                 ++next_seqno;
               }
             }
+
+            return added;
           }
         }
+        return {};
       }
 
     private:
@@ -521,8 +593,6 @@ namespace ccf::historical
     std::unordered_map<ccf::SeqNo, size_t> raw_store_sizes{};
 
     CacheSize soft_store_cache_limit{std::numeric_limits<size_t>::max()};
-    CacheSize soft_store_cache_limit_raw =
-      soft_store_cache_limit / soft_to_raw_ratio;
     CacheSize estimated_store_cache_size{0};
 
     void add_request_ref(SeqNo seq, CompoundHandle handle)
@@ -806,7 +876,11 @@ namespace ccf::historical
                request.supporting_signatures.end());
           if (seqno_in_this_request)
           {
-            request.populate_receipts(seqno);
+            auto added = request.populate_receipts(seqno);
+            for (auto seq : added)
+            {
+              add_request_ref(seq, handle);
+            }
           }
         }
 
@@ -1149,7 +1223,6 @@ namespace ccf::historical
     void set_soft_cache_limit(CacheSize cache_limit)
     {
       soft_store_cache_limit = cache_limit;
-      soft_store_cache_limit_raw = soft_store_cache_limit / soft_to_raw_ratio;
     }
 
     void track_deletes_on_missing_keys(bool track)
@@ -1438,7 +1511,7 @@ namespace ccf::historical
         }
       }
 
-      lru_shrink_to_fit(soft_store_cache_limit_raw);
+      lru_shrink_to_fit(soft_store_cache_limit);
 
       {
         auto it = all_stores.begin();
@@ -1633,5 +1706,10 @@ namespace ccf::historical
     {
       return StateCacheImpl::drop_cached_states(make_compound_handle(handle));
     }
+
+    size_t get_estimated_store_cache_size() override
+    {
+      return StateCacheImpl::get_estimated_store_cache_size();
+    }
   };
 }
diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h
@@ -86,6 +86,19 @@ namespace ccf
   DECLARE_JSON_TYPE(NodeMetrics);
   DECLARE_JSON_REQUIRED_FIELDS(NodeMetrics, sessions);
 
+  struct GetHistoricalCacheInfo
+  {
+    using In = void;
+
+    struct Out
+    {
+      size_t estimated_size;
+    };
+  };
+
+  DECLARE_JSON_TYPE(GetHistoricalCacheInfo::Out);
+  DECLARE_JSON_REQUIRED_FIELDS(GetHistoricalCacheInfo::Out, estimated_size);
+
   struct JavaScriptMetrics
   {
     uint64_t bytecode_size;
@@ -430,7 +443,7 @@ namespace ccf
       openapi_info.description =
         "This API provides public, uncredentialed access to service and node "
         "state.";
-      openapi_info.document_version = "4.16.0";
+      openapi_info.document_version = "4.17.0";
     }
 
     void init_handlers() override
@@ -1862,6 +1875,22 @@ namespace ccf
         .install();
 
       ccf::node::init_file_serving_handlers(*this, context);
+
+      auto historical_cache_info = [this](
+                                     [[maybe_unused]] auto& args,
+                                     [[maybe_unused]] nlohmann::json&&) {
+        GetHistoricalCacheInfo::Out result{};
+        result.estimated_size =
+          this->context.get_historical_state().get_estimated_store_cache_size();
+        return make_success(result);
+      };
+      make_read_only_endpoint(
+        "/historical_cache",
+        HTTP_GET,
+        json_read_only_adapter(historical_cache_info),
+        no_auth_required)
+        .set_auto_schema<GetHistoricalCacheInfo>()
+        .install();
     }
   };
 
diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h
@@ -270,6 +270,11 @@ namespace ccf
     {
       return true;
     }
+
+    size_t get_estimated_store_cache_size()
+    {
+      return 0;
+    }
   };
 
   struct StubNodeContext : public ccf::AbstractNodeContext
diff --git a/src/node/test/historical_queries.cpp b/src/node/test/historical_queries.cpp
diff --git a/tests/historical_query_cache.py b/tests/historical_query_cache.py

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"`
`4`	`4`
`5`	`5`	`[project]`
`6`	`6`	`name = "ccf"`
`7`		`-version = "6.0.25"`
	`7`	`+version = "6.0.26"`
`8`	`8`	`authors = [`
`9`	`9`	`{ name="CCF Team", email="CCF-Sec@microsoft.com" },`
`10`	`10`	`]`
Original file line number	Diff line number	Diff line change
`@@ -270,6 +270,11 @@ namespace ccf`
`270`	`270`	`{`
`271`	`271`	`return true;`
`272`	`272`	`}`
	`273`	`+`
	`274`	`+ size_t get_estimated_store_cache_size()`
	`275`	`+ {`
	`276`	`+ return 0;`
	`277`	`+ }`
`273`	`278`	`};`
`274`	`279`
`275`	`280`	`struct StubNodeContext : public ccf::AbstractNodeContext`