Skip to content

Commit d959118

Browse files
suxiaogang223claude
andcommitted
feat: implement sys.partitions with partition-level aggregation
Replace the stub with actual partition-level file statistics using the AggregateFileStats helper. For each partitioned table, read manifest entries and emit one row per partition with record_count, file_size, file_count, and last_update_time. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 94a2781 commit d959118

1 file changed

Lines changed: 67 additions & 13 deletions

File tree

src/paimon/core/table/system/global_system_tables.cpp

Lines changed: 67 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "paimon/catalog/identifier.h"
3131
#include "paimon/common/data/binary_string.h"
3232
#include "paimon/common/data/generic_row.h"
33+
#include "paimon/common/utils/binary_row_partition_computer.h"
3334
#include "paimon/common/utils/string_utils.h"
3435
#include "paimon/common/utils/path_util.h"
3536
#include "paimon/core/core_options.h"
@@ -152,7 +153,8 @@ Result<std::map<std::string, FileStats>> AggregateFileStats(
152153
PAIMON_ASSIGN_OR_RAISE(
153154
std::unique_ptr<ManifestList> manifest_list,
154155
ManifestList::Create(fs, core_options.GetManifestFormat(),
155-
core_options.GetManifestCompression(), path_factory, pool));
156+
core_options.GetManifestCompression(), path_factory,
157+
core_options.GetCache(), pool));
156158

157159
std::vector<ManifestFileMeta> manifests;
158160
PAIMON_RETURN_NOT_OK(
@@ -183,10 +185,16 @@ Result<std::map<std::string, FileStats>> AggregateFileStats(
183185
}
184186
const auto& file = entry.File();
185187

186-
// Use empty string key for unpartitioned tables
188+
// Convert partition BinaryRow to string representation
187189
std::string partition_key;
188190
if (entry.Partition().GetFieldCount() > 0) {
189-
partition_key = "partitioned";
191+
PAIMON_ASSIGN_OR_RAISE(
192+
partition_key,
193+
BinaryRowPartitionComputer::PartToSimpleString(
194+
partition_schema, entry.Partition(), ",",
195+
/*max_length=*/255,
196+
/*legacy_partition_name_enabled=*/false));
197+
partition_key = "{" + partition_key + "}";
190198
}
191199

192200
auto& stats = result[partition_key];
@@ -458,17 +466,63 @@ Result<std::vector<GenericRow>> PartitionsSystemTable::BuildRows() const {
458466
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> schema, ArrowSchema());
459467
std::vector<GenericRow> rows;
460468

461-
// TODO(suxiaogang223): Implement partition-level aggregation using
462-
// manifest entry reading (similar to FilesSystemTable::BuildRows()
463-
// but grouped by partition). For now, return empty result set.
464-
//
465-
// The implementation should:
466-
// 1. Enumerate all databases and tables
467-
// 2. For each partitioned table, read latest snapshot's manifest entries
468-
// 3. Group DataFileMeta entries by entry.Partition()
469-
// 4. Aggregate: sum(file_size), sum(record_count), count files,
470-
// max(creation_time)
469+
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> databases,
470+
context_.catalog->ListDatabases());
471+
for (const auto& db : databases) {
472+
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> tables,
473+
context_.catalog->ListTables(db));
474+
for (const auto& table : tables) {
475+
Identifier id(db, table);
476+
auto schema_result = context_.catalog->LoadTableSchema(id);
477+
if (!schema_result.ok()) {
478+
continue;
479+
}
480+
auto schema_ptr = schema_result.value();
481+
auto data_schema = std::dynamic_pointer_cast<DataSchema>(schema_ptr);
482+
if (!data_schema) {
483+
continue;
484+
}
471485

486+
// Only emit rows for partitioned tables
487+
if (data_schema->PartitionKeys().empty()) {
488+
continue;
489+
}
490+
491+
// Get table path and aggregate file stats by partition
492+
auto table_path_result = context_.catalog->GetTableLocation(id);
493+
if (!table_path_result.ok()) {
494+
continue;
495+
}
496+
std::string table_path = table_path_result.value();
497+
498+
auto file_stats_result =
499+
AggregateFileStats(context_.fs, table_path, data_schema->Options());
500+
if (!file_stats_result.ok()) {
501+
continue;
502+
}
503+
504+
auto& stats_map = file_stats_result.value();
505+
for (const auto& [partition_key, stats] : stats_map) {
506+
if (stats.file_count == 0) {
507+
continue;
508+
}
509+
GenericRow row(schema->num_fields());
510+
row.SetField(0, std::string_view(db));
511+
row.SetField(1, std::string_view(table));
512+
row.SetField(2, partition_key.empty()
513+
? VariantType(NullType())
514+
: VariantType(StringValue(partition_key)));
515+
row.SetField(3, VariantType(stats.record_count));
516+
row.SetField(4, VariantType(stats.file_size_in_bytes));
517+
row.SetField(5, VariantType(stats.file_count));
518+
row.SetField(6, stats.last_file_creation_time_millis > 0
519+
? VariantType(Timestamp::FromEpochMillis(
520+
stats.last_file_creation_time_millis))
521+
: VariantType(NullType()));
522+
rows.push_back(std::move(row));
523+
}
524+
}
525+
}
472526
return rows;
473527
}
474528

0 commit comments

Comments
 (0)