Skip to content

Commit 00ee7dd

Browse files
suxiaogang223claude
suxiaogang223
and
claude
committed
test: add integration tests for global system tables
Add 4 tests to SystemTableReadInteTest: - TestReadGlobalCatalogOptions: verifies sys.catalog_options schema and content - TestReadGlobalAllTableOptions: verifies sys.all_table_options with table options - TestReadGlobalTables: verifies sys.tables schema, table_type, partitioned, pk - TestReadGlobalPartitions: verifies sys.partitions returns empty for unpartitioned Tests use a ReadGlobalSystemTable helper that creates the GlobalSystemTableContext with a proper Catalog pointer. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent daffd92 commit 00ee7dd

2 files changed

Lines changed: 245 additions & 8 deletions

File tree

‎src/paimon/core/table/system/global_system_tables.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -319,10 +319,10 @@ Result<std::vector<GenericRow>> AllTableOptionsSystemTable::BuildRows() const {
319319
}
320320
for (const auto& [key, value] : data_schema->Options()) {
321321
GenericRow row(schema->num_fields());
322-
row.SetField(0, std::string_view(db));
323-
row.SetField(1, std::string_view(table));
324-
row.SetField(2, std::string_view(key));
325-
row.SetField(3, std::string_view(value));
322+
row.SetField(0, StringValue(db));
323+
row.SetField(1, StringValue(table));
324+
row.SetField(2, StringValue(key));
325+
row.SetField(3, StringValue(value));
326326
rows.push_back(std::move(row));
327327
}
328328
}
@@ -393,8 +393,8 @@ Result<std::vector<GenericRow>> TablesSystemTable::BuildRows() const {
393393
}
394394

395395
GenericRow row(schema->num_fields());
396-
row.SetField(0, std::string_view(db));
397-
row.SetField(1, std::string_view(table));
396+
row.SetField(0, StringValue(db));
397+
row.SetField(1, StringValue(table));
398398
row.SetField(2, StringValue(table_type_str));
399399
row.SetField(3, partitioned);
400400
row.SetField(4, primary_keys_str.empty()
@@ -507,8 +507,8 @@ Result<std::vector<GenericRow>> PartitionsSystemTable::BuildRows() const {
507507
continue;
508508
}
509509
GenericRow row(schema->num_fields());
510-
row.SetField(0, std::string_view(db));
511-
row.SetField(1, std::string_view(table));
510+
row.SetField(0, StringValue(db));
511+
row.SetField(1, StringValue(table));
512512
row.SetField(2, partition_key.empty()
513513
? VariantType(NullType())
514514
: VariantType(StringValue(partition_key)));

‎test/inte/read_inte_test.cpp‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17+
#include <algorithm>
1718
#include <cstddef>
1819
#include <cstdint>
1920
#include <cstdlib>
@@ -53,6 +54,7 @@
5354
#include "paimon/data/decimal.h"
5455
#include "paimon/data/timestamp.h"
5556
#include "paimon/defs.h"
57+
#include "paimon/core/table/system/global_system_tables.h"
5658
#include "paimon/fs/file_system.h"
5759
#include "paimon/fs/local/local_file_system.h"
5860
#include "paimon/memory/memory_pool.h"
@@ -3717,4 +3719,239 @@ TEST_P(ReadInteTest, TestSpecificFs) {
37173719
ASSERT_GT(io_count, 0);
37183720
}
37193721

3722+
// =============================================================================
3723+
// Global System Table Tests
3724+
// =============================================================================
3725+
3726+
namespace {
3727+
3728+
Result<SystemTableReadResult> ReadGlobalSystemTable(
3729+
const std::string& table_name, Catalog* catalog,
3730+
const std::shared_ptr<FileSystem>& fs, const std::string& warehouse,
3731+
const std::map<std::string, std::string>& options) {
3732+
GlobalSystemTableContext ctx;
3733+
ctx.catalog = catalog;
3734+
ctx.fs = fs;
3735+
ctx.warehouse = warehouse;
3736+
ctx.catalog_options = options;
3737+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
3738+
GlobalSystemTableLoader::Load(table_name, ctx));
3739+
3740+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> arrow_schema,
3741+
system_table->ArrowSchema());
3742+
3743+
std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name);
3744+
3745+
ScanContextBuilder scan_context_builder(sys_path);
3746+
scan_context_builder.SetOptions(options);
3747+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ScanContext> scan_context,
3748+
scan_context_builder.Finish());
3749+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
3750+
system_table->NewScan(scan_context));
3751+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, table_scan->CreatePlan());
3752+
3753+
ReadContextBuilder read_context_builder(sys_path);
3754+
read_context_builder.SetOptions(options);
3755+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context,
3756+
read_context_builder.Finish());
3757+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
3758+
system_table->NewRead(read_context));
3759+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
3760+
table_read->CreateReader(plan->Splits()));
3761+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> result,
3762+
ReadResultCollector::CollectResult(batch_reader.get()));
3763+
return SystemTableReadResult(std::move(batch_reader), result);
3764+
}
3765+
3766+
} // namespace
3767+
3768+
TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) {
3769+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3770+
{Options::FILE_FORMAT, "orc"},
3771+
{"custom.catalog.option", "test-value"}};
3772+
auto dir = UniqueTestDirectory::Create();
3773+
ASSERT_TRUE(dir);
3774+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3775+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3776+
ASSERT_OK_AND_ASSIGN(auto result,
3777+
ReadGlobalSystemTable("catalog_options", catalog.get(),
3778+
catalog->GetFileSystem(), warehouse, options));
3779+
auto struct_array = SingleStructChunk(result);
3780+
ASSERT_TRUE(struct_array);
3781+
auto key_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3782+
auto value_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3783+
ASSERT_TRUE(key_array);
3784+
ASSERT_TRUE(value_array);
3785+
3786+
// Build a map from the result
3787+
std::map<std::string, std::string> result_map;
3788+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3789+
result_map[key_array->GetString(i)] = value_array->GetString(i);
3790+
}
3791+
ASSERT_EQ(result_map["file-system"], "local");
3792+
ASSERT_EQ(result_map["file.format"], "orc");
3793+
}
3794+
3795+
TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {
3796+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3797+
{Options::FILE_FORMAT, "orc"},
3798+
{Options::MANIFEST_FORMAT, "orc"},
3799+
{"table.option.custom", "my-value"}};
3800+
auto dir = UniqueTestDirectory::Create();
3801+
ASSERT_TRUE(dir);
3802+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3803+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3804+
// Create a database and table
3805+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3806+
auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())});
3807+
::ArrowSchema schema;
3808+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3809+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3810+
/*partition_keys=*/{}, /*primary_keys=*/{}, options,
3811+
/*ignore_if_exists=*/false));
3812+
ArrowSchemaRelease(&schema);
3813+
3814+
// Verify basic enumeration works
3815+
ASSERT_OK_AND_ASSIGN(auto dbs, catalog->ListDatabases());
3816+
ASSERT_TRUE(std::find(dbs.begin(), dbs.end(), "test_db") != dbs.end());
3817+
ASSERT_OK_AND_ASSIGN(auto tbls, catalog->ListTables("test_db"));
3818+
ASSERT_TRUE(std::find(tbls.begin(), tbls.end(), "test_tbl") != tbls.end());
3819+
3820+
// Verify schema loads and has Options
3821+
ASSERT_OK_AND_ASSIGN(auto loaded_schema,
3822+
catalog->LoadTableSchema(Identifier("test_db", "test_tbl")));
3823+
auto ds = std::dynamic_pointer_cast<DataSchema>(loaded_schema);
3824+
ASSERT_TRUE(ds != nullptr) << "LoadTableSchema did not return DataSchema";
3825+
ASSERT_FALSE(ds->Options().empty()) << "Table schema has no options";
3826+
3827+
// Directly test BuildRows
3828+
{
3829+
GlobalSystemTableContext ctx;
3830+
ctx.catalog = catalog.get();
3831+
ctx.fs = catalog->GetFileSystem();
3832+
ctx.warehouse = warehouse;
3833+
ctx.catalog_options = options;
3834+
AllTableOptionsSystemTable table(ctx);
3835+
ASSERT_OK_AND_ASSIGN(auto rows, table.BuildRows());
3836+
ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row";
3837+
}
3838+
3839+
ASSERT_OK_AND_ASSIGN(auto result,
3840+
ReadGlobalSystemTable("all_table_options", catalog.get(),
3841+
catalog->GetFileSystem(), warehouse, options));
3842+
auto struct_array = SingleStructChunk(result);
3843+
ASSERT_TRUE(struct_array);
3844+
ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows";
3845+
3846+
auto db_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3847+
auto tbl_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3848+
auto key_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(2));
3849+
auto val_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(3));
3850+
ASSERT_TRUE(db_array);
3851+
ASSERT_TRUE(tbl_array);
3852+
ASSERT_TRUE(key_array);
3853+
ASSERT_TRUE(val_array);
3854+
3855+
// Verify that our table's options appear in the result
3856+
bool found_db = false;
3857+
bool found_format = false;
3858+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3859+
auto db_name = std::string(db_array->GetString(i));
3860+
auto tbl_name = std::string(tbl_array->GetString(i));
3861+
if (db_name == "test_db" && tbl_name == "test_tbl") {
3862+
found_db = true;
3863+
auto key_str = std::string(key_array->GetString(i));
3864+
if (key_str == "file.format") {
3865+
EXPECT_EQ(std::string(val_array->GetString(i)), "orc");
3866+
found_format = true;
3867+
}
3868+
}
3869+
}
3870+
ASSERT_TRUE(found_db) << "test_db.test_tbl not found in sys.all_table_options";
3871+
ASSERT_TRUE(found_format) << "file.format option not found in sys.all_table_options";
3872+
}
3873+
3874+
TEST(SystemTableReadInteTest, TestReadGlobalTables) {
3875+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3876+
{Options::FILE_FORMAT, "orc"},
3877+
{Options::MANIFEST_FORMAT, "orc"},
3878+
{Options::BUCKET, "1"}};
3879+
auto dir = UniqueTestDirectory::Create();
3880+
ASSERT_TRUE(dir);
3881+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3882+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3883+
auto fs = catalog->GetFileSystem();
3884+
3885+
// Create a database and a PK table
3886+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3887+
auto typed_schema = arrow::schema({
3888+
arrow::field("pk", arrow::utf8()),
3889+
arrow::field("v", arrow::int32()),
3890+
});
3891+
::ArrowSchema schema;
3892+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3893+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3894+
/*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options,
3895+
/*ignore_if_exists=*/false));
3896+
ArrowSchemaRelease(&schema);
3897+
3898+
ASSERT_OK_AND_ASSIGN(auto result,
3899+
ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options));
3900+
auto struct_array = SingleStructChunk(result);
3901+
ASSERT_TRUE(struct_array);
3902+
ASSERT_GE(struct_array->length(), 1);
3903+
3904+
auto db_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3905+
auto tbl_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3906+
auto type_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(2));
3907+
auto part_array = std::dynamic_pointer_cast<arrow::BooleanArray>(struct_array->field(3));
3908+
auto pk_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(4));
3909+
ASSERT_TRUE(db_array);
3910+
ASSERT_TRUE(tbl_array);
3911+
ASSERT_TRUE(type_array);
3912+
ASSERT_TRUE(part_array);
3913+
ASSERT_TRUE(pk_array);
3914+
3915+
// Find our table by table name
3916+
bool found = false;
3917+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3918+
if (std::string(tbl_array->GetString(i)) == "test_tbl") {
3919+
EXPECT_EQ(std::string(db_array->GetString(i)), "test_db");
3920+
EXPECT_EQ(std::string(type_array->GetString(i)), "MANAGED");
3921+
EXPECT_FALSE(part_array->Value(i));
3922+
// pk is stored as BinaryString; check not null
3923+
EXPECT_FALSE(pk_array->IsNull(i));
3924+
found = true;
3925+
}
3926+
}
3927+
ASSERT_TRUE(found) << "table not found in sys.tables";
3928+
}
3929+
3930+
TEST(SystemTableReadInteTest, TestReadGlobalPartitions) {
3931+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3932+
{Options::FILE_FORMAT, "orc"},
3933+
{Options::MANIFEST_FORMAT, "orc"}};
3934+
auto dir = UniqueTestDirectory::Create();
3935+
ASSERT_TRUE(dir);
3936+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3937+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3938+
auto fs = catalog->GetFileSystem();
3939+
3940+
// Create a database and an unpartitioned table (no partitions → empty result)
3941+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3942+
auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())});
3943+
::ArrowSchema schema;
3944+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3945+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3946+
/*partition_keys=*/{}, /*primary_keys=*/{}, options,
3947+
/*ignore_if_exists=*/false));
3948+
ArrowSchemaRelease(&schema);
3949+
3950+
// Unpartitioned tables are skipped by sys.partitions → empty result.
3951+
// CollectResult returns null shared_ptr when result is empty.
3952+
ASSERT_OK_AND_ASSIGN(auto part_result,
3953+
ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options));
3954+
ASSERT_FALSE(part_result.array) << "expected null array for empty partitions result";
3955+
}
3956+
37203957
} // namespace paimon::test

0 commit comments

Comments
 (0)