Skip to content

Commit c5d3439

Browse files
suxiaogang223claude
suxiaogang223
and
claude
committed
test: add integration tests for global system tables
Add 4 tests to SystemTableReadInteTest: - TestReadGlobalCatalogOptions: verifies sys.catalog_options schema and content - TestReadGlobalAllTableOptions: verifies sys.all_table_options with table options - TestReadGlobalTables: verifies sys.tables schema, table_type, partitioned, pk - TestReadGlobalPartitions: verifies sys.partitions returns empty for unpartitioned Tests use a ReadGlobalSystemTable helper that creates the GlobalSystemTableContext with a proper Catalog pointer. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent d959118 commit c5d3439

2 files changed

Lines changed: 245 additions & 8 deletions

File tree

‎src/paimon/core/table/system/global_system_tables.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -319,10 +319,10 @@ Result<std::vector<GenericRow>> AllTableOptionsSystemTable::BuildRows() const {
319319
}
320320
for (const auto& [key, value] : data_schema->Options()) {
321321
GenericRow row(schema->num_fields());
322-
row.SetField(0, std::string_view(db));
323-
row.SetField(1, std::string_view(table));
324-
row.SetField(2, std::string_view(key));
325-
row.SetField(3, std::string_view(value));
322+
row.SetField(0, StringValue(db));
323+
row.SetField(1, StringValue(table));
324+
row.SetField(2, StringValue(key));
325+
row.SetField(3, StringValue(value));
326326
rows.push_back(std::move(row));
327327
}
328328
}
@@ -393,8 +393,8 @@ Result<std::vector<GenericRow>> TablesSystemTable::BuildRows() const {
393393
}
394394

395395
GenericRow row(schema->num_fields());
396-
row.SetField(0, std::string_view(db));
397-
row.SetField(1, std::string_view(table));
396+
row.SetField(0, StringValue(db));
397+
row.SetField(1, StringValue(table));
398398
row.SetField(2, StringValue(table_type_str));
399399
row.SetField(3, partitioned);
400400
row.SetField(4, primary_keys_str.empty()
@@ -507,8 +507,8 @@ Result<std::vector<GenericRow>> PartitionsSystemTable::BuildRows() const {
507507
continue;
508508
}
509509
GenericRow row(schema->num_fields());
510-
row.SetField(0, std::string_view(db));
511-
row.SetField(1, std::string_view(table));
510+
row.SetField(0, StringValue(db));
511+
row.SetField(1, StringValue(table));
512512
row.SetField(2, partition_key.empty()
513513
? VariantType(NullType())
514514
: VariantType(StringValue(partition_key)));

‎test/inte/read_inte_test.cpp‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17+
#include <algorithm>
1718
#include <cstddef>
1819
#include <cstdint>
1920
#include <cstdlib>
@@ -53,6 +54,7 @@
5354
#include "paimon/data/decimal.h"
5455
#include "paimon/data/timestamp.h"
5556
#include "paimon/defs.h"
57+
#include "paimon/core/table/system/global_system_tables.h"
5658
#include "paimon/fs/file_system.h"
5759
#include "paimon/fs/local/local_file_system.h"
5860
#include "paimon/memory/memory_pool.h"
@@ -3353,4 +3355,239 @@ TEST_P(ReadInteTest, TestSpecificFs) {
33533355
ASSERT_GT(io_count, 0);
33543356
}
33553357

3358+
// =============================================================================
3359+
// Global System Table Tests
3360+
// =============================================================================
3361+
3362+
namespace {
3363+
3364+
Result<SystemTableReadResult> ReadGlobalSystemTable(
3365+
const std::string& table_name, Catalog* catalog,
3366+
const std::shared_ptr<FileSystem>& fs, const std::string& warehouse,
3367+
const std::map<std::string, std::string>& options) {
3368+
GlobalSystemTableContext ctx;
3369+
ctx.catalog = catalog;
3370+
ctx.fs = fs;
3371+
ctx.warehouse = warehouse;
3372+
ctx.catalog_options = options;
3373+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
3374+
GlobalSystemTableLoader::Load(table_name, ctx));
3375+
3376+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> arrow_schema,
3377+
system_table->ArrowSchema());
3378+
3379+
std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name);
3380+
3381+
ScanContextBuilder scan_context_builder(sys_path);
3382+
scan_context_builder.SetOptions(options);
3383+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ScanContext> scan_context,
3384+
scan_context_builder.Finish());
3385+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableScan> table_scan,
3386+
system_table->NewScan(scan_context));
3387+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Plan> plan, table_scan->CreatePlan());
3388+
3389+
ReadContextBuilder read_context_builder(sys_path);
3390+
read_context_builder.SetOptions(options);
3391+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ReadContext> read_context,
3392+
read_context_builder.Finish());
3393+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
3394+
system_table->NewRead(read_context));
3395+
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
3396+
table_read->CreateReader(plan->Splits()));
3397+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::ChunkedArray> result,
3398+
ReadResultCollector::CollectResult(batch_reader.get()));
3399+
return SystemTableReadResult(std::move(batch_reader), result);
3400+
}
3401+
3402+
} // namespace
3403+
3404+
TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) {
3405+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3406+
{Options::FILE_FORMAT, "orc"},
3407+
{"custom.catalog.option", "test-value"}};
3408+
auto dir = UniqueTestDirectory::Create();
3409+
ASSERT_TRUE(dir);
3410+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3411+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3412+
ASSERT_OK_AND_ASSIGN(auto result,
3413+
ReadGlobalSystemTable("catalog_options", catalog.get(),
3414+
catalog->GetFileSystem(), warehouse, options));
3415+
auto struct_array = SingleStructChunk(result);
3416+
ASSERT_TRUE(struct_array);
3417+
auto key_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3418+
auto value_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3419+
ASSERT_TRUE(key_array);
3420+
ASSERT_TRUE(value_array);
3421+
3422+
// Build a map from the result
3423+
std::map<std::string, std::string> result_map;
3424+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3425+
result_map[key_array->GetString(i)] = value_array->GetString(i);
3426+
}
3427+
ASSERT_EQ(result_map["file-system"], "local");
3428+
ASSERT_EQ(result_map["file.format"], "orc");
3429+
}
3430+
3431+
TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {
3432+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3433+
{Options::FILE_FORMAT, "orc"},
3434+
{Options::MANIFEST_FORMAT, "orc"},
3435+
{"table.option.custom", "my-value"}};
3436+
auto dir = UniqueTestDirectory::Create();
3437+
ASSERT_TRUE(dir);
3438+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3439+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3440+
// Create a database and table
3441+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3442+
auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())});
3443+
::ArrowSchema schema;
3444+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3445+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3446+
/*partition_keys=*/{}, /*primary_keys=*/{}, options,
3447+
/*ignore_if_exists=*/false));
3448+
ArrowSchemaRelease(&schema);
3449+
3450+
// Verify basic enumeration works
3451+
ASSERT_OK_AND_ASSIGN(auto dbs, catalog->ListDatabases());
3452+
ASSERT_TRUE(std::find(dbs.begin(), dbs.end(), "test_db") != dbs.end());
3453+
ASSERT_OK_AND_ASSIGN(auto tbls, catalog->ListTables("test_db"));
3454+
ASSERT_TRUE(std::find(tbls.begin(), tbls.end(), "test_tbl") != tbls.end());
3455+
3456+
// Verify schema loads and has Options
3457+
ASSERT_OK_AND_ASSIGN(auto loaded_schema,
3458+
catalog->LoadTableSchema(Identifier("test_db", "test_tbl")));
3459+
auto ds = std::dynamic_pointer_cast<DataSchema>(loaded_schema);
3460+
ASSERT_TRUE(ds != nullptr) << "LoadTableSchema did not return DataSchema";
3461+
ASSERT_FALSE(ds->Options().empty()) << "Table schema has no options";
3462+
3463+
// Directly test BuildRows
3464+
{
3465+
GlobalSystemTableContext ctx;
3466+
ctx.catalog = catalog.get();
3467+
ctx.fs = catalog->GetFileSystem();
3468+
ctx.warehouse = warehouse;
3469+
ctx.catalog_options = options;
3470+
AllTableOptionsSystemTable table(ctx);
3471+
ASSERT_OK_AND_ASSIGN(auto rows, table.BuildRows());
3472+
ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row";
3473+
}
3474+
3475+
ASSERT_OK_AND_ASSIGN(auto result,
3476+
ReadGlobalSystemTable("all_table_options", catalog.get(),
3477+
catalog->GetFileSystem(), warehouse, options));
3478+
auto struct_array = SingleStructChunk(result);
3479+
ASSERT_TRUE(struct_array);
3480+
ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows";
3481+
3482+
auto db_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3483+
auto tbl_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3484+
auto key_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(2));
3485+
auto val_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(3));
3486+
ASSERT_TRUE(db_array);
3487+
ASSERT_TRUE(tbl_array);
3488+
ASSERT_TRUE(key_array);
3489+
ASSERT_TRUE(val_array);
3490+
3491+
// Verify that our table's options appear in the result
3492+
bool found_db = false;
3493+
bool found_format = false;
3494+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3495+
auto db_name = std::string(db_array->GetString(i));
3496+
auto tbl_name = std::string(tbl_array->GetString(i));
3497+
if (db_name == "test_db" && tbl_name == "test_tbl") {
3498+
found_db = true;
3499+
auto key_str = std::string(key_array->GetString(i));
3500+
if (key_str == "file.format") {
3501+
EXPECT_EQ(std::string(val_array->GetString(i)), "orc");
3502+
found_format = true;
3503+
}
3504+
}
3505+
}
3506+
ASSERT_TRUE(found_db) << "test_db.test_tbl not found in sys.all_table_options";
3507+
ASSERT_TRUE(found_format) << "file.format option not found in sys.all_table_options";
3508+
}
3509+
3510+
TEST(SystemTableReadInteTest, TestReadGlobalTables) {
3511+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3512+
{Options::FILE_FORMAT, "orc"},
3513+
{Options::MANIFEST_FORMAT, "orc"},
3514+
{Options::BUCKET, "1"}};
3515+
auto dir = UniqueTestDirectory::Create();
3516+
ASSERT_TRUE(dir);
3517+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3518+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3519+
auto fs = catalog->GetFileSystem();
3520+
3521+
// Create a database and a PK table
3522+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3523+
auto typed_schema = arrow::schema({
3524+
arrow::field("pk", arrow::utf8()),
3525+
arrow::field("v", arrow::int32()),
3526+
});
3527+
::ArrowSchema schema;
3528+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3529+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3530+
/*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options,
3531+
/*ignore_if_exists=*/false));
3532+
ArrowSchemaRelease(&schema);
3533+
3534+
ASSERT_OK_AND_ASSIGN(auto result,
3535+
ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options));
3536+
auto struct_array = SingleStructChunk(result);
3537+
ASSERT_TRUE(struct_array);
3538+
ASSERT_GE(struct_array->length(), 1);
3539+
3540+
auto db_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(0));
3541+
auto tbl_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(1));
3542+
auto type_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(2));
3543+
auto part_array = std::dynamic_pointer_cast<arrow::BooleanArray>(struct_array->field(3));
3544+
auto pk_array = std::dynamic_pointer_cast<arrow::StringArray>(struct_array->field(4));
3545+
ASSERT_TRUE(db_array);
3546+
ASSERT_TRUE(tbl_array);
3547+
ASSERT_TRUE(type_array);
3548+
ASSERT_TRUE(part_array);
3549+
ASSERT_TRUE(pk_array);
3550+
3551+
// Find our table by table name
3552+
bool found = false;
3553+
for (int64_t i = 0; i < struct_array->length(); ++i) {
3554+
if (std::string(tbl_array->GetString(i)) == "test_tbl") {
3555+
EXPECT_EQ(std::string(db_array->GetString(i)), "test_db");
3556+
EXPECT_EQ(std::string(type_array->GetString(i)), "MANAGED");
3557+
EXPECT_FALSE(part_array->Value(i));
3558+
// pk is stored as BinaryString; check not null
3559+
EXPECT_FALSE(pk_array->IsNull(i));
3560+
found = true;
3561+
}
3562+
}
3563+
ASSERT_TRUE(found) << "table not found in sys.tables";
3564+
}
3565+
3566+
TEST(SystemTableReadInteTest, TestReadGlobalPartitions) {
3567+
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM, "local"},
3568+
{Options::FILE_FORMAT, "orc"},
3569+
{Options::MANIFEST_FORMAT, "orc"}};
3570+
auto dir = UniqueTestDirectory::Create();
3571+
ASSERT_TRUE(dir);
3572+
std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse");
3573+
ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options));
3574+
auto fs = catalog->GetFileSystem();
3575+
3576+
// Create a database and an unpartitioned table (no partitions → empty result)
3577+
ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false));
3578+
auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())});
3579+
::ArrowSchema schema;
3580+
ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok());
3581+
ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema,
3582+
/*partition_keys=*/{}, /*primary_keys=*/{}, options,
3583+
/*ignore_if_exists=*/false));
3584+
ArrowSchemaRelease(&schema);
3585+
3586+
// Unpartitioned tables are skipped by sys.partitions → empty result.
3587+
// CollectResult returns null shared_ptr when result is empty.
3588+
ASSERT_OK_AND_ASSIGN(auto part_result,
3589+
ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options));
3590+
ASSERT_FALSE(part_result.array) << "expected null array for empty partitions result";
3591+
}
3592+
33563593
} // namespace paimon::test

0 commit comments

Comments
 (0)