Add condition cache query profiling

src/include/query_condition_cache_filter.hpp

@@ -3,18 +3,31 @@
 #include "query_condition_cache_state.hpp"
 
 #include "duckdb/function/scalar_function.hpp"
+#include "duckdb/function/table/table_scan.hpp"
 #include "duckdb/planner/expression.hpp"
 #include "duckdb/planner/filter/expression_filter.hpp"
 #include "duckdb/storage/statistics/numeric_stats.hpp"
 
 namespace duckdb {
 
+struct ConditionCacheTableScanBindData : public TableScanBindData {
+	ConditionCacheTableScanBindData(TableCatalogEntry &table, shared_ptr<ConditionCacheProfileInfo> profile_info_p);
+
+	shared_ptr<ConditionCacheProfileInfo> profile_info;
+
+	unique_ptr<FunctionData> Copy() const override;
+};
+
+InsertionOrderPreservingMap<string> ConditionCacheDynamicToString(TableFunctionDynamicToStringInput &input);
+
 // Holds a shared_ptr to the ConditionCacheEntry so the filter function
 // and CheckStatistics can query the cache via ConditionCacheEntry's thread-safe API.
 struct ConditionCacheFilterBindData : public FunctionData {
 	shared_ptr<ConditionCacheEntry> cache_entry;
+	shared_ptr<ConditionCacheProfileInfo> profile_info;
 
-	explicit ConditionCacheFilterBindData(shared_ptr<ConditionCacheEntry> entry);
+	ConditionCacheFilterBindData(shared_ptr<ConditionCacheEntry> entry,
+	                             shared_ptr<ConditionCacheProfileInfo> profile_info_p = nullptr);
 
 	unique_ptr<FunctionData> Copy() const override;
 	bool Equals(const FunctionData &other_p) const override;
@@ -42,8 +55,10 @@ void ConditionCacheFilterFn(DataChunk &args, ExpressionState &state, Vector &res
 class CacheExpressionFilter : public ExpressionFilter {
 public:
 	shared_ptr<ConditionCacheEntry> cache_entry;
+	shared_ptr<ConditionCacheProfileInfo> profile_info;
 
-	CacheExpressionFilter(unique_ptr<Expression> expr, shared_ptr<ConditionCacheEntry> entry);
+	CacheExpressionFilter(unique_ptr<Expression> expr, shared_ptr<ConditionCacheEntry> entry,
+	                      shared_ptr<ConditionCacheProfileInfo> profile_info_p = nullptr);
 
 	FilterPropagateResult CheckStatistics(BaseStatistics &stats) const override;
 	unique_ptr<TableFilter> Copy() const override;

src/include/query_condition_cache_optimizer.hpp

@@ -10,14 +10,19 @@ namespace duckdb {
 class DuckTableEntry;
 class LogicalGet;
 
+struct PendingCacheApplyData {
+	shared_ptr<ConditionCacheEntry> cache_entry;
+	shared_ptr<ConditionCacheProfileInfo> profile_info;
+};
+
 // Query-scoped state for passing cache entries between pre-optimize and post-optimize phases.
 // Stored in ClientContext::registered_state; automatically cleared on QueryEnd.
 struct CacheOptimizerQueryState : public ClientContextState {
 	static constexpr const char *NAME = "qcc_optimizer_state";
 
 	// Maps table_index -> cache entry for tables matched during pre-optimize.
 	// Consumed by post-optimize to inject cache filters.
-	unordered_map<idx_t, shared_ptr<ConditionCacheEntry>> cache_apply_pending;
+	unordered_map<idx_t, PendingCacheApplyData> cache_apply_pending;
 
 	void QueryEnd(ClientContext &context, optional_ptr<ErrorData> error) override {
 		cache_apply_pending.clear();
@@ -51,8 +56,8 @@ class QueryConditionCacheOptimizer : public OptimizerExtension {
 	                             CacheOptimizerQueryState &state);
 
 	// Inject a rowid-backed cache filter into a LogicalGet while preserving its visible output.
-	static void InjectCacheFilter(ClientContext &context, LogicalGet &get,
-	                              const shared_ptr<ConditionCacheEntry> &entry);
+	static void InjectCacheFilter(ClientContext &context, LogicalGet &get, const shared_ptr<ConditionCacheEntry> &entry,
+	                              const shared_ptr<ConditionCacheProfileInfo> &profile_info);
 };
 
 } // namespace duckdb

src/include/query_condition_cache_state.hpp

@@ -60,6 +60,17 @@ struct CacheEntryStats {
 	idx_t total_row_groups;
 };
 
+struct ConditionCacheProfileInfo {
+	atomic<bool> initial_lookup_hit {false};
+	atomic<bool> built_this_query {false};
+
+	string predicate_hash;
+	idx_t cached_row_groups = 0;
+	idx_t qualifying_vectors = 0;
+	idx_t total_vectors = 0;
+	idx_t total_rows = 0;
+};
+
 // A single cache entry: the bitvectors for one (table, predicate) combination.
 struct ConditionCacheEntry : public ObjectCacheEntry {
 	static string ObjectType() {

src/query_condition_cache_filter.cpp

@@ -2,22 +2,72 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/numeric_utils.hpp"
+#include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 
 namespace duckdb {
 
-ConditionCacheFilterBindData::ConditionCacheFilterBindData(shared_ptr<ConditionCacheEntry> entry)
-    : cache_entry(std::move(entry)) {
+namespace {
+
+string GetConditionCacheStatus(const ConditionCacheProfileInfo &profile_info) {
+	if (profile_info.built_this_query.load()) {
+		return "MISS -> BUILT";
+	}
+	if (profile_info.initial_lookup_hit.load()) {
+		return "HIT";
+	}
+	return "MISS";
+}
+
+} // namespace
+
+ConditionCacheTableScanBindData::ConditionCacheTableScanBindData(TableCatalogEntry &table,
+                                                                 shared_ptr<ConditionCacheProfileInfo> profile_info_p)
+    : TableScanBindData(table), profile_info(std::move(profile_info_p)) {
+}
+
+unique_ptr<FunctionData> ConditionCacheTableScanBindData::Copy() const {
+	auto result = make_uniq<ConditionCacheTableScanBindData>(table, profile_info);
+	result->is_index_scan = is_index_scan;
+	result->is_create_index = is_create_index;
+	result->column_ids = column_ids;
+	result->order_options = order_options ? make_uniq<RowGroupOrderOptions>(*order_options) : nullptr;
+	return std::move(result);
+}
+
+InsertionOrderPreservingMap<string> ConditionCacheDynamicToString(TableFunctionDynamicToStringInput &input) {
+	InsertionOrderPreservingMap<string> result;
+	if (!input.bind_data) {
+		return result;
+	}
+
+	auto &bind_data = input.bind_data->Cast<ConditionCacheTableScanBindData>();
+	if (!bind_data.profile_info) {
+		return result;
+	}
+
+	auto &profile_info = *bind_data.profile_info;
+	result["Condition Cache"] = GetConditionCacheStatus(profile_info);
+	result["Condition Cache Predicate Hash"] = profile_info.predicate_hash;
+	result["Condition Cache Cached Row Groups"] = to_string(profile_info.cached_row_groups);
+	result["Condition Cache Qualifying Vectors"] =
+	    StringUtil::Format("%llu/%llu", profile_info.qualifying_vectors, profile_info.total_vectors);
+	return result;
+}
+
+ConditionCacheFilterBindData::ConditionCacheFilterBindData(shared_ptr<ConditionCacheEntry> entry,
+                                                           shared_ptr<ConditionCacheProfileInfo> profile_info_p)
+    : cache_entry(std::move(entry)), profile_info(std::move(profile_info_p)) {
 }
 
 unique_ptr<FunctionData> ConditionCacheFilterBindData::Copy() const {
-	return make_uniq<ConditionCacheFilterBindData>(cache_entry);
+	return make_uniq<ConditionCacheFilterBindData>(cache_entry, profile_info);
 }
 
 bool ConditionCacheFilterBindData::Equals(const FunctionData &other_p) const {
 	auto &other = other_p.Cast<ConditionCacheFilterBindData>();
-	return cache_entry == other.cache_entry;
+	return cache_entry == other.cache_entry && profile_info == other.profile_info;
 }
 
 unique_ptr<FunctionData> ConditionCacheFilterBind(ClientContext &context, ScalarFunction &bound_function,
@@ -61,8 +111,9 @@ void ConditionCacheFilterFn(DataChunk &args, ExpressionState &state, Vector &res
 	result.Reference(Value::BOOLEAN(passes));
 }
 
-CacheExpressionFilter::CacheExpressionFilter(unique_ptr<Expression> expr_p, shared_ptr<ConditionCacheEntry> entry)
-    : ExpressionFilter(std::move(expr_p)), cache_entry(std::move(entry)) {
+CacheExpressionFilter::CacheExpressionFilter(unique_ptr<Expression> expr_p, shared_ptr<ConditionCacheEntry> entry,
+                                             shared_ptr<ConditionCacheProfileInfo> profile_info_p)
+    : ExpressionFilter(std::move(expr_p)), cache_entry(std::move(entry)), profile_info(std::move(profile_info_p)) {
 }
 
 FilterPropagateResult CacheExpressionFilter::CheckStatistics(BaseStatistics &stats) const {
@@ -83,7 +134,7 @@ FilterPropagateResult CacheExpressionFilter::CheckStatistics(BaseStatistics &sta
 }
 
 unique_ptr<TableFilter> CacheExpressionFilter::Copy() const {
-	return make_uniq<CacheExpressionFilter>(expr->Copy(), cache_entry);
+	return make_uniq<CacheExpressionFilter>(expr->Copy(), cache_entry, profile_info);
 }
 
 } // namespace duckdb

src/query_condition_cache_optimizer.cpp

@@ -6,6 +6,7 @@
 #include "query_condition_cache_state.hpp"
 
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/common/string_util.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
@@ -99,21 +100,32 @@ void QueryConditionCacheOptimizer::PreOptimizeWalk(ClientContext &context, uniqu
 		return;
 	}
 
+	auto profile_info = make_shared_ptr<ConditionCacheProfileInfo>();
 	auto store = ConditionCacheStore::GetOrCreate(context);
 	auto entry = store->Lookup(context, key);
 	store->RecordAccess(entry != nullptr);
+	profile_info->initial_lookup_hit.store(entry != nullptr);
 
 	if (!entry) {
 		// TODO: Consider building cache in the background and syncing later
 		// to avoid blocking the first query.
 		entry = BuildCacheForPredicate(context, filter.expressions, get);
 		if (entry) {
+			profile_info->built_this_query.store(true);
 			store->Upsert(context, key, entry);
 		}
 	}
 
 	if (entry) {
-		state.cache_apply_pending[get.table_index] = std::move(entry);
+		auto total_rows = storage.GetTotalRows();
+		auto entry_stats = entry->ComputeStats(total_rows);
+		profile_info->predicate_hash = StringUtil::Format("%llu", Hash(key.filter_key.c_str()));
+		profile_info->cached_row_groups = entry->RowGroupCount();
+		profile_info->qualifying_vectors = entry_stats.qualifying_vectors;
+		profile_info->total_vectors = entry_stats.total_vectors;
+		profile_info->total_rows = total_rows;
+
+		state.cache_apply_pending[get.table_index] = {entry, profile_info};
 	}
 }
 
@@ -178,12 +190,13 @@ void QueryConditionCacheOptimizer::PostOptimizeWalk(ClientContext &context, uniq
 		return;
 	}
 
-	InjectCacheFilter(context, get, entry->second);
+	InjectCacheFilter(context, get, entry->second.cache_entry, entry->second.profile_info);
 	state.cache_apply_pending.erase(entry);
 }
 
 void QueryConditionCacheOptimizer::InjectCacheFilter(ClientContext &context, LogicalGet &get,
-                                                     const shared_ptr<ConditionCacheEntry> &entry) {
+                                                     const shared_ptr<ConditionCacheEntry> &entry,
+                                                     const shared_ptr<ConditionCacheProfileInfo> &profile_info) {
 	auto &column_ids = get.GetMutableColumnIds();
 	bool has_row_id = false;
 	for (const auto &column_id : column_ids) {
@@ -202,15 +215,26 @@ void QueryConditionCacheOptimizer::InjectCacheFilter(ClientContext &context, Log
 		column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID);
 	}
 
+	auto &table_scan_bind_data = get.bind_data->Cast<TableScanBindData>();
+	auto profiled_bind_data = make_uniq<ConditionCacheTableScanBindData>(table_scan_bind_data.table, profile_info);
+	profiled_bind_data->is_index_scan = table_scan_bind_data.is_index_scan;
+	profiled_bind_data->is_create_index = table_scan_bind_data.is_create_index;
+	profiled_bind_data->column_ids = table_scan_bind_data.column_ids;
+	profiled_bind_data->order_options = table_scan_bind_data.order_options
+	                                        ? make_uniq<RowGroupOrderOptions>(*table_scan_bind_data.order_options)
+	                                        : nullptr;
+	get.bind_data = std::move(profiled_bind_data);
+	get.function.dynamic_to_string = ConditionCacheDynamicToString;
+
 	vector<unique_ptr<Expression>> children;
 	children.push_back(make_uniq<BoundReferenceExpression>(LogicalType {LogicalTypeId::BIGINT}, 0U));
 
-	auto filter_expr =
-	    make_uniq<BoundFunctionExpression>(LogicalType {LogicalTypeId::BOOLEAN}, ConditionCacheFilterFunction(),
-	                                       std::move(children), make_uniq<ConditionCacheFilterBindData>(entry));
+	auto filter_expr = make_uniq<BoundFunctionExpression>(LogicalType {LogicalTypeId::BOOLEAN},
+	                                                      ConditionCacheFilterFunction(), std::move(children),
+	                                                      make_uniq<ConditionCacheFilterBindData>(entry, profile_info));
 
 	get.table_filters.PushFilter(ColumnIndex(COLUMN_IDENTIFIER_ROW_ID),
-	                             make_uniq<CacheExpressionFilter>(std::move(filter_expr), entry));
+	                             make_uniq<CacheExpressionFilter>(std::move(filter_expr), entry, profile_info));
 }
 
 void QueryConditionCacheOptimizer::OptimizeFunction(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan) {

test/sql/condition_cache_explain.test

@@ -0,0 +1,39 @@
+# name: test/sql/condition_cache_explain.test
+# description: Test condition cache stats in EXPLAIN ANALYZE and JSON profiling
+# group: [sql]
+
+require query_condition_cache
+
+statement ok
+CREATE TABLE t AS SELECT i AS id FROM range(500000) t(i);
+
+query II
+EXPLAIN (ANALYZE, FORMAT JSON) SELECT count(*) FROM t WHERE id < 3000;
+----
+analyzed_plan	<REGEX>:.*"Condition Cache": "MISS -> BUILT".*"Condition Cache Predicate Hash": "[0-9]+".*"Condition Cache Cached Row Groups": "5".*"Condition Cache Qualifying Vectors": "2/245".*
+
+statement ok
+PRAGMA enable_profiling='json';
+
+statement ok
+PRAGMA profiling_output='__TEST_DIR__/condition_cache_profile.json';
+
+statement ok
+SELECT count(*) FROM t WHERE id < 3000;
+
+statement ok
+PRAGMA disable_profiling;
+
+query I
+SELECT contains(profile_json, 'Condition Cache')
+       AND contains(profile_json, 'HIT')
+       AND contains(profile_json, 'Condition Cache Predicate Hash')
+       AND contains(profile_json, 'Condition Cache Cached Row Groups')
+       AND contains(profile_json, 'Condition Cache Qualifying Vectors')
+       AND contains(profile_json, '2/245')
+FROM (
+    SELECT string_agg(json, '\n') AS profile_json
+    FROM read_csv('__TEST_DIR__/condition_cache_profile.json', columns={'json': 'VARCHAR'}, sep='🦆')
+);
+----
+true