Skip to content

Commit 4d898e9

Browse files
mkleenmkleen
authored
feat: Expose cache hits in statistics_cache function (#23253)
## Which issue does this PR close? - Closes None. ## Rationale for this change Follow up to #22613. Cache hits are now supported for all memory-limiting caches. Therefore it makes sense to expose them in the `statistics_cache` function the same way the `metadata_cache` function does it already. ## What changes are included in this PR? - Add cache hits to the `statistics_cache` function - Tests - Adapt documentation for the `statistics_cache` function ## Are these changes tested? Yes. ## Are there any user-facing changes? Yes, the `statistics_cache` function supports now cache hits but no breaking changes.
1 parent 0d2c791 commit 4d898e9

3 files changed

Lines changed: 53 additions & 16 deletions

File tree

‎datafusion-cli/src/functions.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,6 +649,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
649649
Field::new("num_columns", DataType::UInt64, false),
650650
Field::new("table_size_bytes", DataType::Utf8, false),
651651
Field::new("statistics_size_bytes", DataType::UInt64, false),
652+
Field::new("hits", DataType::UInt64, false),
652653
]));
653654

654655
// construct record batch from metadata
@@ -662,6 +663,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
662663
let mut num_columns_arr = vec![];
663664
let mut table_size_bytes_arr = vec![];
664665
let mut statistics_size_bytes_arr = vec![];
666+
let mut hits_arr = vec![];
665667

666668
if let Some(file_statistics_cache) = self.cache_manager.get_file_statistic_cache()
667669
{
@@ -686,6 +688,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
686688
.heap_size(&mut DFHeapSizeCtx::default())
687689
as u64,
688690
);
691+
hits_arr.push(entry.hits as u64);
689692
}
690693
}
691694

@@ -702,6 +705,7 @@ impl TableFunctionImpl for StatisticsCacheFunc {
702705
Arc::new(UInt64Array::from(num_columns_arr)),
703706
Arc::new(StringArray::from(table_size_bytes_arr)),
704707
Arc::new(UInt64Array::from(statistics_size_bytes_arr)),
708+
Arc::new(UInt64Array::from(hits_arr)),
705709
],
706710
)?;
707711

‎datafusion-cli/src/main.rs‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -711,17 +711,48 @@ mod tests {
711711
.await?;
712712
}
713713

714-
let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, table_size_bytes from statistics_cache() order by filename";
714+
let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
715715
let df = ctx.sql(sql).await?;
716716
let rbs = df.collect().await?;
717-
assert_snapshot!(batches_to_string(&rbs),@r"
718-
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
719-
| filename | table | file_size_bytes | num_rows | num_columns | table_size_bytes |
720-
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
721-
| alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | Absent |
722-
| alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | Absent |
723-
| lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | Absent |
724-
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------------------+
717+
assert_snapshot!(batches_to_string(&rbs),@"
718+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
719+
| filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
720+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
721+
| alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 0 | Absent |
722+
| alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
723+
| lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 0 | Absent |
724+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
725+
");
726+
727+
// increase the number of hits
728+
ctx.sql("select * from alltypes_plain")
729+
.await?
730+
.collect()
731+
.await?;
732+
ctx.sql("select * from alltypes_plain")
733+
.await?
734+
.collect()
735+
.await?;
736+
ctx.sql("select * from alltypes_plain")
737+
.await?
738+
.collect()
739+
.await?;
740+
ctx.sql("select * from lz4_raw_compressed_larger")
741+
.await?
742+
.collect()
743+
.await?;
744+
745+
let sql = "SELECT split_part(path, '/', -1) as filename, table, file_size_bytes, num_rows, num_columns, hits, table_size_bytes from statistics_cache() order by filename";
746+
let df = ctx.sql(sql).await?;
747+
let rbs = df.collect().await?;
748+
assert_snapshot!(batches_to_string(&rbs),@"
749+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
750+
| filename | table | file_size_bytes | num_rows | num_columns | hits | table_size_bytes |
751+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
752+
| alltypes_plain.parquet | alltypes_plain | 1851 | Exact(8) | 11 | 3 | Absent |
753+
| alltypes_tiny_pages.parquet | alltypes_tiny_pages | 454233 | Exact(7300) | 13 | 0 | Absent |
754+
| lz4_raw_compressed_larger.parquet | lz4_raw_compressed_larger | 380836 | Exact(10000) | 1 | 1 | Absent |
755+
+-----------------------------------+---------------------------+-----------------+--------------+-------------+------+------------------+
725756
");
726757

727758
Ok(())

‎docs/source/user-guide/cli/functions.md‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ The columns of the returned table are:
168168
| num_rows | Utf8 | Number of rows in the table |
169169
| num_columns | UInt64 | Number of columns in the table |
170170
| table_size_bytes | Utf8 | Size of the table, in bytes |
171+
| hits | UInt64 | Number of times the cached file statistics has been accessed |
171172
| statistics_size_bytes | UInt64 | Size of the cached statistics in memory |
172173

173174
## `list_files_cache`
@@ -200,13 +201,14 @@ location 's3://overturemaps-us-west-2/release/2025-12-17.0/theme=base/type=infra
200201
```
201202

202203
The columns of the returned table are:
203-
| column_name | data_type | Description |
204-
| ------------------- | ------------ | ----------------------------------------------------------------------------------------- |
205-
| table | Utf8 | Name of the table |
206-
| path | Utf8 | File path relative to the object store / filesystem root |
207-
| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
208-
| expires_in | Duration(ms) | Last modified time of the file |
209-
| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
204+
205+
| column_name | data_type | Description |
206+
| ------------------- | ------------ | ------------------------------------------------------------------- |
207+
| table | Utf8 | Name of the table |
208+
| path | Utf8 | File path relative to the object store / filesystem root |
209+
| metadata_size_bytes | UInt64 | Size of the cached metadata in memory (not its thrift encoded form) |
210+
| expires_in | Duration(ms) | Last modified time of the file |
211+
| metadata_list | List(Struct) | List of metadatas, one for each file under the path. |
210212

211213
A metadata struct in the metadata_list contains the following fields:
212214

0 commit comments

Comments
 (0)