diff --git a/src/include/query_condition_cache_filter.hpp b/src/include/query_condition_cache_filter.hpp index 304a7d5..31c24e9 100644 --- a/src/include/query_condition_cache_filter.hpp +++ b/src/include/query_condition_cache_filter.hpp @@ -3,18 +3,31 @@ #include "query_condition_cache_state.hpp" #include "duckdb/function/scalar_function.hpp" +#include "duckdb/function/table/table_scan.hpp" #include "duckdb/planner/expression.hpp" #include "duckdb/planner/filter/expression_filter.hpp" #include "duckdb/storage/statistics/numeric_stats.hpp" namespace duckdb { +struct ConditionCacheTableScanBindData : public TableScanBindData { + ConditionCacheTableScanBindData(TableCatalogEntry &table, shared_ptr profile_info_p); + + shared_ptr profile_info; + + unique_ptr Copy() const override; +}; + +InsertionOrderPreservingMap ConditionCacheDynamicToString(TableFunctionDynamicToStringInput &input); + // Holds a shared_ptr to the ConditionCacheEntry so the filter function // and CheckStatistics can query the cache via ConditionCacheEntry's thread-safe API. struct ConditionCacheFilterBindData : public FunctionData { shared_ptr cache_entry; + shared_ptr profile_info; - explicit ConditionCacheFilterBindData(shared_ptr entry); + ConditionCacheFilterBindData(shared_ptr entry, + shared_ptr profile_info_p = nullptr); unique_ptr Copy() const override; bool Equals(const FunctionData &other_p) const override; @@ -42,8 +55,10 @@ void ConditionCacheFilterFn(DataChunk &args, ExpressionState &state, Vector &res class CacheExpressionFilter : public ExpressionFilter { public: shared_ptr cache_entry; + shared_ptr profile_info; - CacheExpressionFilter(unique_ptr expr, shared_ptr entry); + CacheExpressionFilter(unique_ptr expr, shared_ptr entry, + shared_ptr profile_info_p = nullptr); FilterPropagateResult CheckStatistics(BaseStatistics &stats) const override; unique_ptr Copy() const override; diff --git a/src/include/query_condition_cache_optimizer.hpp b/src/include/query_condition_cache_optimizer.hpp index 221646a..4f5cc1b 100644 --- a/src/include/query_condition_cache_optimizer.hpp +++ b/src/include/query_condition_cache_optimizer.hpp @@ -10,6 +10,11 @@ namespace duckdb { class DuckTableEntry; class LogicalGet; +struct PendingCacheApplyData { + shared_ptr cache_entry; + shared_ptr profile_info; +}; + // Query-scoped state for passing cache entries between pre-optimize and post-optimize phases. // Stored in ClientContext::registered_state; automatically cleared on QueryEnd. struct CacheOptimizerQueryState : public ClientContextState { @@ -17,7 +22,7 @@ struct CacheOptimizerQueryState : public ClientContextState { // Maps table_index -> cache entry for tables matched during pre-optimize. // Consumed by post-optimize to inject cache filters. - unordered_map> cache_apply_pending; + unordered_map cache_apply_pending; void QueryEnd(ClientContext &context, optional_ptr error) override { cache_apply_pending.clear(); @@ -51,8 +56,8 @@ class QueryConditionCacheOptimizer : public OptimizerExtension { CacheOptimizerQueryState &state); // Inject a rowid-backed cache filter into a LogicalGet while preserving its visible output. - static void InjectCacheFilter(ClientContext &context, LogicalGet &get, - const shared_ptr &entry); + static void InjectCacheFilter(ClientContext &context, LogicalGet &get, const shared_ptr &entry, + const shared_ptr &profile_info); }; } // namespace duckdb diff --git a/src/include/query_condition_cache_state.hpp b/src/include/query_condition_cache_state.hpp index 4220127..ab36e9f 100644 --- a/src/include/query_condition_cache_state.hpp +++ b/src/include/query_condition_cache_state.hpp @@ -60,6 +60,17 @@ struct CacheEntryStats { idx_t total_row_groups; }; +struct ConditionCacheProfileInfo { + atomic initial_lookup_hit {false}; + atomic built_this_query {false}; + + string predicate_hash; + idx_t cached_row_groups = 0; + idx_t qualifying_vectors = 0; + idx_t total_vectors = 0; + idx_t total_rows = 0; +}; + // A single cache entry: the bitvectors for one (table, predicate) combination. struct ConditionCacheEntry : public ObjectCacheEntry { static string ObjectType() { diff --git a/src/query_condition_cache_filter.cpp b/src/query_condition_cache_filter.cpp index 5bc9989..c63feb9 100644 --- a/src/query_condition_cache_filter.cpp +++ b/src/query_condition_cache_filter.cpp @@ -2,22 +2,72 @@ #include "duckdb/common/assert.hpp" #include "duckdb/common/numeric_utils.hpp" +#include "duckdb/common/string_util.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" namespace duckdb { -ConditionCacheFilterBindData::ConditionCacheFilterBindData(shared_ptr entry) - : cache_entry(std::move(entry)) { +namespace { + +string GetConditionCacheStatus(const ConditionCacheProfileInfo &profile_info) { + if (profile_info.built_this_query.load()) { + return "MISS -> BUILT"; + } + if (profile_info.initial_lookup_hit.load()) { + return "HIT"; + } + return "MISS"; +} + +} // namespace + +ConditionCacheTableScanBindData::ConditionCacheTableScanBindData(TableCatalogEntry &table, + shared_ptr profile_info_p) + : TableScanBindData(table), profile_info(std::move(profile_info_p)) { +} + +unique_ptr ConditionCacheTableScanBindData::Copy() const { + auto result = make_uniq(table, profile_info); + result->is_index_scan = is_index_scan; + result->is_create_index = is_create_index; + result->column_ids = column_ids; + result->order_options = order_options ? make_uniq(*order_options) : nullptr; + return std::move(result); +} + +InsertionOrderPreservingMap ConditionCacheDynamicToString(TableFunctionDynamicToStringInput &input) { + InsertionOrderPreservingMap result; + if (!input.bind_data) { + return result; + } + + auto &bind_data = input.bind_data->Cast(); + if (!bind_data.profile_info) { + return result; + } + + auto &profile_info = *bind_data.profile_info; + result["Condition Cache"] = GetConditionCacheStatus(profile_info); + result["Condition Cache Predicate Hash"] = profile_info.predicate_hash; + result["Condition Cache Cached Row Groups"] = to_string(profile_info.cached_row_groups); + result["Condition Cache Qualifying Vectors"] = + StringUtil::Format("%llu/%llu", profile_info.qualifying_vectors, profile_info.total_vectors); + return result; +} + +ConditionCacheFilterBindData::ConditionCacheFilterBindData(shared_ptr entry, + shared_ptr profile_info_p) + : cache_entry(std::move(entry)), profile_info(std::move(profile_info_p)) { } unique_ptr ConditionCacheFilterBindData::Copy() const { - return make_uniq(cache_entry); + return make_uniq(cache_entry, profile_info); } bool ConditionCacheFilterBindData::Equals(const FunctionData &other_p) const { auto &other = other_p.Cast(); - return cache_entry == other.cache_entry; + return cache_entry == other.cache_entry && profile_info == other.profile_info; } unique_ptr ConditionCacheFilterBind(ClientContext &context, ScalarFunction &bound_function, @@ -61,8 +111,9 @@ void ConditionCacheFilterFn(DataChunk &args, ExpressionState &state, Vector &res result.Reference(Value::BOOLEAN(passes)); } -CacheExpressionFilter::CacheExpressionFilter(unique_ptr expr_p, shared_ptr entry) - : ExpressionFilter(std::move(expr_p)), cache_entry(std::move(entry)) { +CacheExpressionFilter::CacheExpressionFilter(unique_ptr expr_p, shared_ptr entry, + shared_ptr profile_info_p) + : ExpressionFilter(std::move(expr_p)), cache_entry(std::move(entry)), profile_info(std::move(profile_info_p)) { } FilterPropagateResult CacheExpressionFilter::CheckStatistics(BaseStatistics &stats) const { @@ -83,7 +134,7 @@ FilterPropagateResult CacheExpressionFilter::CheckStatistics(BaseStatistics &sta } unique_ptr CacheExpressionFilter::Copy() const { - return make_uniq(expr->Copy(), cache_entry); + return make_uniq(expr->Copy(), cache_entry, profile_info); } } // namespace duckdb diff --git a/src/query_condition_cache_optimizer.cpp b/src/query_condition_cache_optimizer.cpp index ed61357..a2eca1c 100644 --- a/src/query_condition_cache_optimizer.cpp +++ b/src/query_condition_cache_optimizer.cpp @@ -6,6 +6,7 @@ #include "query_condition_cache_state.hpp" #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp" +#include "duckdb/common/string_util.hpp" #include "duckdb/common/vector.hpp" #include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" @@ -99,21 +100,32 @@ void QueryConditionCacheOptimizer::PreOptimizeWalk(ClientContext &context, uniqu return; } + auto profile_info = make_shared_ptr(); auto store = ConditionCacheStore::GetOrCreate(context); auto entry = store->Lookup(context, key); store->RecordAccess(entry != nullptr); + profile_info->initial_lookup_hit.store(entry != nullptr); if (!entry) { // TODO: Consider building cache in the background and syncing later // to avoid blocking the first query. entry = BuildCacheForPredicate(context, filter.expressions, get); if (entry) { + profile_info->built_this_query.store(true); store->Upsert(context, key, entry); } } if (entry) { - state.cache_apply_pending[get.table_index] = std::move(entry); + auto total_rows = storage.GetTotalRows(); + auto entry_stats = entry->ComputeStats(total_rows); + profile_info->predicate_hash = StringUtil::Format("%llu", Hash(key.filter_key.c_str())); + profile_info->cached_row_groups = entry->RowGroupCount(); + profile_info->qualifying_vectors = entry_stats.qualifying_vectors; + profile_info->total_vectors = entry_stats.total_vectors; + profile_info->total_rows = total_rows; + + state.cache_apply_pending[get.table_index] = {entry, profile_info}; } } @@ -178,12 +190,13 @@ void QueryConditionCacheOptimizer::PostOptimizeWalk(ClientContext &context, uniq return; } - InjectCacheFilter(context, get, entry->second); + InjectCacheFilter(context, get, entry->second.cache_entry, entry->second.profile_info); state.cache_apply_pending.erase(entry); } void QueryConditionCacheOptimizer::InjectCacheFilter(ClientContext &context, LogicalGet &get, - const shared_ptr &entry) { + const shared_ptr &entry, + const shared_ptr &profile_info) { auto &column_ids = get.GetMutableColumnIds(); bool has_row_id = false; for (const auto &column_id : column_ids) { @@ -202,15 +215,26 @@ void QueryConditionCacheOptimizer::InjectCacheFilter(ClientContext &context, Log column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID); } + auto &table_scan_bind_data = get.bind_data->Cast(); + auto profiled_bind_data = make_uniq(table_scan_bind_data.table, profile_info); + profiled_bind_data->is_index_scan = table_scan_bind_data.is_index_scan; + profiled_bind_data->is_create_index = table_scan_bind_data.is_create_index; + profiled_bind_data->column_ids = table_scan_bind_data.column_ids; + profiled_bind_data->order_options = table_scan_bind_data.order_options + ? make_uniq(*table_scan_bind_data.order_options) + : nullptr; + get.bind_data = std::move(profiled_bind_data); + get.function.dynamic_to_string = ConditionCacheDynamicToString; + vector> children; children.push_back(make_uniq(LogicalType {LogicalTypeId::BIGINT}, 0U)); - auto filter_expr = - make_uniq(LogicalType {LogicalTypeId::BOOLEAN}, ConditionCacheFilterFunction(), - std::move(children), make_uniq(entry)); + auto filter_expr = make_uniq(LogicalType {LogicalTypeId::BOOLEAN}, + ConditionCacheFilterFunction(), std::move(children), + make_uniq(entry, profile_info)); get.table_filters.PushFilter(ColumnIndex(COLUMN_IDENTIFIER_ROW_ID), - make_uniq(std::move(filter_expr), entry)); + make_uniq(std::move(filter_expr), entry, profile_info)); } void QueryConditionCacheOptimizer::OptimizeFunction(OptimizerExtensionInput &input, unique_ptr &plan) { diff --git a/test/sql/condition_cache_explain.test b/test/sql/condition_cache_explain.test new file mode 100644 index 0000000..acc8956 --- /dev/null +++ b/test/sql/condition_cache_explain.test @@ -0,0 +1,39 @@ +# name: test/sql/condition_cache_explain.test +# description: Test condition cache stats in EXPLAIN ANALYZE and JSON profiling +# group: [sql] + +require query_condition_cache + +statement ok +CREATE TABLE t AS SELECT i AS id FROM range(500000) t(i); + +query II +EXPLAIN (ANALYZE, FORMAT JSON) SELECT count(*) FROM t WHERE id < 3000; +---- +analyzed_plan :.*"Condition Cache": "MISS -> BUILT".*"Condition Cache Predicate Hash": "[0-9]+".*"Condition Cache Cached Row Groups": "5".*"Condition Cache Qualifying Vectors": "2/245".* + +statement ok +PRAGMA enable_profiling='json'; + +statement ok +PRAGMA profiling_output='__TEST_DIR__/condition_cache_profile.json'; + +statement ok +SELECT count(*) FROM t WHERE id < 3000; + +statement ok +PRAGMA disable_profiling; + +query I +SELECT contains(profile_json, 'Condition Cache') + AND contains(profile_json, 'HIT') + AND contains(profile_json, 'Condition Cache Predicate Hash') + AND contains(profile_json, 'Condition Cache Cached Row Groups') + AND contains(profile_json, 'Condition Cache Qualifying Vectors') + AND contains(profile_json, '2/245') +FROM ( + SELECT string_agg(json, '\n') AS profile_json + FROM read_csv('__TEST_DIR__/condition_cache_profile.json', columns={'json': 'VARCHAR'}, sep='🦆') +); +---- +true diff --git a/test/unittest/test_filter.cpp b/test/unittest/test_filter.cpp index be5f68c..f5de4ab 100644 --- a/test/unittest/test_filter.cpp +++ b/test/unittest/test_filter.cpp @@ -4,6 +4,7 @@ #include "duckdb/main/connection.hpp" #include "duckdb/main/database.hpp" +#include "duckdb/main/profiling_node.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/filter/expression_filter.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" @@ -28,6 +29,23 @@ optional_ptr FindLogicalGet(LogicalOperator &op) { return nullptr; } +optional_ptr FindConditionCacheProfilingNode(ProfilingNode &node) { + auto &info = node.GetProfilingInfo(); + if (info.metrics.find(MetricType::EXTRA_INFO) != info.metrics.end()) { + auto extra_info = info.GetMetricValue>(MetricType::EXTRA_INFO); + if (extra_info.find("Condition Cache") != extra_info.end()) { + return node; + } + } + for (idx_t i = 0; i < node.GetChildCount(); ++i) { + auto child = FindConditionCacheProfilingNode(*node.GetChild(i)); + if (child) { + return child; + } + } + return nullptr; +} + } // namespace TEST_CASE("CacheExpressionFilter - CheckStatistics", "[query_condition_cache]") { @@ -94,6 +112,11 @@ TEST_CASE("Optimizer injects cache filter into LogicalGet", "[query_condition_ca auto &bind_data = function_expr.bind_info->Cast(); REQUIRE(bind_data.cache_entry != nullptr); + REQUIRE(bind_data.profile_info != nullptr); + REQUIRE(get->function.dynamic_to_string == ConditionCacheDynamicToString); + + auto &scan_bind_data = get->bind_data->Cast(); + REQUIRE(scan_bind_data.profile_info != nullptr); return plan; }; @@ -129,4 +152,58 @@ TEST_CASE("Optimizer injects cache filter into LogicalGet", "[query_condition_ca REQUIRE(get->table_filters.filters.find(COLUMN_IDENTIFIER_ROW_ID) == get->table_filters.filters.end()); } } + +TEST_CASE("Condition cache profiling extra info is surfaced in explain analyze and JSON profiling", + "[query_condition_cache][profile]") { + DuckDB db(nullptr); + Connection con(db); + + REQUIRE_NO_FAIL(con.Query("LOAD query_condition_cache")); + REQUIRE_NO_FAIL(con.Query("CREATE TABLE t AS SELECT i AS id FROM range(500000) t(i)")); + + auto explain_result = con.Query("EXPLAIN ANALYZE SELECT count(*) FROM t WHERE id < 3000"); + REQUIRE(explain_result); + REQUIRE(!explain_result->HasError()); + + auto explain_text = explain_result->GetValue(1, 0).ToString(); + REQUIRE(explain_text.find("Condition Cache") != string::npos); + REQUIRE(explain_text.find("MISS -> BUILT") != string::npos); + + con.EnableProfiling(); + con.context->config.emit_profiler_output = false; + + auto query_result = con.Query("SELECT count(*) FROM t WHERE id < 3000"); + REQUIRE(query_result); + REQUIRE(!query_result->HasError()); + REQUIRE(query_result->GetValue(0, 0).GetValue() == 3000); + + auto profiling_root = con.GetProfilingTree(); + REQUIRE(profiling_root); + + auto condition_cache_node = FindConditionCacheProfilingNode(*profiling_root); + REQUIRE(condition_cache_node); + + auto extra_info = condition_cache_node->GetProfilingInfo().GetMetricValue>( + MetricType::EXTRA_INFO); + + auto status = extra_info.find("Condition Cache"); + REQUIRE(status != extra_info.end()); + REQUIRE(status->second == "HIT"); + + auto predicate_hash = extra_info.find("Condition Cache Predicate Hash"); + REQUIRE(predicate_hash != extra_info.end()); + REQUIRE(!predicate_hash->second.empty()); + + auto cached_row_groups = extra_info.find("Condition Cache Cached Row Groups"); + REQUIRE(cached_row_groups != extra_info.end()); + REQUIRE(cached_row_groups->second == "5"); + + auto qualifying_vectors = extra_info.find("Condition Cache Qualifying Vectors"); + REQUIRE(qualifying_vectors != extra_info.end()); + REQUIRE(qualifying_vectors->second == "2/245"); + + auto json_profile = con.GetProfilingInformation(ProfilerPrintFormat::JSON); + REQUIRE(json_profile.find("Condition Cache") != string::npos); + REQUIRE(json_profile.find("HIT") != string::npos); +} } // namespace duckdb