|
30 | 30 | #include "paimon/catalog/identifier.h" |
31 | 31 | #include "paimon/common/data/binary_string.h" |
32 | 32 | #include "paimon/common/data/generic_row.h" |
| 33 | +#include "paimon/common/utils/binary_row_partition_computer.h" |
33 | 34 | #include "paimon/common/utils/string_utils.h" |
34 | 35 | #include "paimon/common/utils/path_util.h" |
35 | 36 | #include "paimon/core/core_options.h" |
@@ -152,7 +153,8 @@ Result<std::map<std::string, FileStats>> AggregateFileStats( |
152 | 153 | PAIMON_ASSIGN_OR_RAISE( |
153 | 154 | std::unique_ptr<ManifestList> manifest_list, |
154 | 155 | ManifestList::Create(fs, core_options.GetManifestFormat(), |
155 | | - core_options.GetManifestCompression(), path_factory, pool)); |
| 156 | + core_options.GetManifestCompression(), path_factory, |
| 157 | + core_options.GetCache(), pool)); |
156 | 158 |
|
157 | 159 | std::vector<ManifestFileMeta> manifests; |
158 | 160 | PAIMON_RETURN_NOT_OK( |
@@ -183,10 +185,16 @@ Result<std::map<std::string, FileStats>> AggregateFileStats( |
183 | 185 | } |
184 | 186 | const auto& file = entry.File(); |
185 | 187 |
|
186 | | - // Use empty string key for unpartitioned tables |
| 188 | + // Convert partition BinaryRow to string representation |
187 | 189 | std::string partition_key; |
188 | 190 | if (entry.Partition().GetFieldCount() > 0) { |
189 | | - partition_key = "partitioned"; |
| 191 | + PAIMON_ASSIGN_OR_RAISE( |
| 192 | + partition_key, |
| 193 | + BinaryRowPartitionComputer::PartToSimpleString( |
| 194 | + partition_schema, entry.Partition(), ",", |
| 195 | + /*max_length=*/255, |
| 196 | + /*legacy_partition_name_enabled=*/false)); |
| 197 | + partition_key = "{" + partition_key + "}"; |
190 | 198 | } |
191 | 199 |
|
192 | 200 | auto& stats = result[partition_key]; |
@@ -458,17 +466,63 @@ Result<std::vector<GenericRow>> PartitionsSystemTable::BuildRows() const { |
458 | 466 | PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> schema, ArrowSchema()); |
459 | 467 | std::vector<GenericRow> rows; |
460 | 468 |
|
461 | | - // TODO(suxiaogang223): Implement partition-level aggregation using |
462 | | - // manifest entry reading (similar to FilesSystemTable::BuildRows() |
463 | | - // but grouped by partition). For now, return empty result set. |
464 | | - // |
465 | | - // The implementation should: |
466 | | - // 1. Enumerate all databases and tables |
467 | | - // 2. For each partitioned table, read latest snapshot's manifest entries |
468 | | - // 3. Group DataFileMeta entries by entry.Partition() |
469 | | - // 4. Aggregate: sum(file_size), sum(record_count), count files, |
470 | | - // max(creation_time) |
| 469 | + PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> databases, |
| 470 | + context_.catalog->ListDatabases()); |
| 471 | + for (const auto& db : databases) { |
| 472 | + PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> tables, |
| 473 | + context_.catalog->ListTables(db)); |
| 474 | + for (const auto& table : tables) { |
| 475 | + Identifier id(db, table); |
| 476 | + auto schema_result = context_.catalog->LoadTableSchema(id); |
| 477 | + if (!schema_result.ok()) { |
| 478 | + continue; |
| 479 | + } |
| 480 | + auto schema_ptr = schema_result.value(); |
| 481 | + auto data_schema = std::dynamic_pointer_cast<DataSchema>(schema_ptr); |
| 482 | + if (!data_schema) { |
| 483 | + continue; |
| 484 | + } |
471 | 485 |
|
| 486 | + // Only emit rows for partitioned tables |
| 487 | + if (data_schema->PartitionKeys().empty()) { |
| 488 | + continue; |
| 489 | + } |
| 490 | + |
| 491 | + // Get table path and aggregate file stats by partition |
| 492 | + auto table_path_result = context_.catalog->GetTableLocation(id); |
| 493 | + if (!table_path_result.ok()) { |
| 494 | + continue; |
| 495 | + } |
| 496 | + std::string table_path = table_path_result.value(); |
| 497 | + |
| 498 | + auto file_stats_result = |
| 499 | + AggregateFileStats(context_.fs, table_path, data_schema->Options()); |
| 500 | + if (!file_stats_result.ok()) { |
| 501 | + continue; |
| 502 | + } |
| 503 | + |
| 504 | + auto& stats_map = file_stats_result.value(); |
| 505 | + for (const auto& [partition_key, stats] : stats_map) { |
| 506 | + if (stats.file_count == 0) { |
| 507 | + continue; |
| 508 | + } |
| 509 | + GenericRow row(schema->num_fields()); |
| 510 | + row.SetField(0, std::string_view(db)); |
| 511 | + row.SetField(1, std::string_view(table)); |
| 512 | + row.SetField(2, partition_key.empty() |
| 513 | + ? VariantType(NullType()) |
| 514 | + : VariantType(StringValue(partition_key))); |
| 515 | + row.SetField(3, VariantType(stats.record_count)); |
| 516 | + row.SetField(4, VariantType(stats.file_size_in_bytes)); |
| 517 | + row.SetField(5, VariantType(stats.file_count)); |
| 518 | + row.SetField(6, stats.last_file_creation_time_millis > 0 |
| 519 | + ? VariantType(Timestamp::FromEpochMillis( |
| 520 | + stats.last_file_creation_time_millis)) |
| 521 | + : VariantType(NullType())); |
| 522 | + rows.push_back(std::move(row)); |
| 523 | + } |
| 524 | + } |
| 525 | + } |
472 | 526 | return rows; |
473 | 527 | } |
474 | 528 |
|
|
0 commit comments