Skip to content

Commit 00ce9cb

Browse files
suxiaogang223claude
andcommitted
feat: add global system tables framework under sys database
- Add GetOptions() to Catalog interface for catalog-level config access - FileSystemCatalog stores and exposes catalog_options - New GlobalSystemTableLoader with independent registry for sys tables - Implement sys.catalog_options, sys.all_table_options, sys.tables - Stub for sys.partitions (manifest aggregation to follow) - Extend SystemTablePath with is_global flag - TryParsePath detects sys/ paths for TableScan/TableRead routing - FileSystemCatalog handles sys database in ListTables, DatabaseExists, TableExists, LoadTableSchema Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent e547186 commit 00ce9cb

9 files changed

Lines changed: 553 additions & 8 deletions

File tree

include/paimon/catalog/catalog.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ class PAIMON_EXPORT Catalog {
183183
/// @return A shared pointer to the file system instance.
184184
virtual std::shared_ptr<FileSystem> GetFileSystem() const = 0;
185185

186+
/// Returns the catalog-level options that were passed during catalog creation.
187+
///
188+
/// @return A const reference to the map of catalog options (key-value pairs).
189+
virtual const std::map<std::string, std::string>& GetOptions() const = 0;
190+
186191
/// Loads the latest schema of a specified table.
187192
///
188193
/// @note System tables will not be supported.

src/paimon/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ set(PAIMON_CORE_SRCS
328328
core/table/source/data_evolution_batch_scan.cpp
329329
core/table/system/audit_log_system_table.cpp
330330
core/table/system/binlog_system_table.cpp
331+
core/table/system/global_system_tables.cpp
331332
core/table/system/in_memory_system_table.cpp
332333
core/table/system/metadata_system_tables.cpp
333334
core/table/system/system_table.cpp

src/paimon/core/catalog/catalog.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ Result<std::unique_ptr<Catalog>> Catalog::Create(const std::string& root_path,
3232
const std::map<std::string, std::string>& options,
3333
const std::shared_ptr<FileSystem>& file_system) {
3434
PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system));
35-
return std::make_unique<FileSystemCatalog>(core_options.GetFileSystem(), root_path);
35+
return std::make_unique<FileSystemCatalog>(core_options.GetFileSystem(), root_path, options);
3636
}
3737

3838
} // namespace paimon

src/paimon/core/catalog/file_system_catalog.cpp

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "paimon/common/utils/string_utils.h"
3232
#include "paimon/core/core_options.h"
3333
#include "paimon/core/snapshot.h"
34+
#include "paimon/core/table/system/global_system_tables.h"
3435
#include "paimon/core/table/system/system_table.h"
3536
#include "paimon/core/table/system/system_table_schema.h"
3637
#include "paimon/core/utils/branch_manager.h"
@@ -47,8 +48,12 @@ struct ArrowSchema;
4748

4849
namespace paimon {
4950
FileSystemCatalog::FileSystemCatalog(const std::shared_ptr<FileSystem>& fs,
50-
const std::string& warehouse)
51-
: fs_(fs), warehouse_(warehouse), logger_(Logger::GetLogger("FileSystemCatalog")) {}
51+
const std::string& warehouse,
52+
const std::map<std::string, std::string>& catalog_options)
53+
: fs_(fs),
54+
warehouse_(warehouse),
55+
catalog_options_(catalog_options),
56+
logger_(Logger::GetLogger("FileSystemCatalog")) {}
5257

5358
Status FileSystemCatalog::CreateDatabase(const std::string& db_name,
5459
const std::map<std::string, std::string>& options,
@@ -88,13 +93,16 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name,
8893

8994
Result<bool> FileSystemCatalog::DatabaseExists(const std::string& db_name) const {
9095
if (IsSystemDatabase(db_name)) {
91-
return Status::NotImplemented(
92-
"do not support checking DatabaseExists for system database.");
96+
return true;
9397
}
9498
return fs_->Exists(NewDatabasePath(warehouse_, db_name));
9599
}
96100

97101
Result<bool> FileSystemCatalog::TableExists(const Identifier& identifier) const {
102+
// Handle sys database global tables
103+
if (IsSystemDatabase(identifier.GetDatabaseName())) {
104+
return GlobalSystemTableLoader::IsSupported(identifier.GetTableName());
105+
}
98106
PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable());
99107
if (is_system_table) {
100108
PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> system_table_name,
@@ -184,6 +192,10 @@ std::shared_ptr<FileSystem> FileSystemCatalog::GetFileSystem() const {
184192
return fs_;
185193
}
186194

195+
const std::map<std::string, std::string>& FileSystemCatalog::GetOptions() const {
196+
return catalog_options_;
197+
}
198+
187199
bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) {
188200
return db_name == SYSTEM_DATABASE_NAME;
189201
}
@@ -228,7 +240,7 @@ Result<std::vector<std::string>> FileSystemCatalog::ListDatabases() const {
228240

229241
Result<std::vector<std::string>> FileSystemCatalog::ListTables(const std::string& db_name) const {
230242
if (IsSystemDatabase(db_name)) {
231-
return Status::NotImplemented("do not support listing tables for system database.");
243+
return GlobalSystemTableLoader::GetSupportedTableNames();
232244
}
233245
std::string database_path = NewDatabasePath(warehouse_, db_name);
234246
std::vector<std::unique_ptr<BasicFileStatus>> file_status_list;
@@ -261,6 +273,23 @@ Result<bool> FileSystemCatalog::TableExistsInFileSystem(const std::string& table
261273

262274
Result<std::shared_ptr<Schema>> FileSystemCatalog::LoadTableSchema(
263275
const Identifier& identifier) const {
276+
// Handle sys database global tables
277+
if (IsSystemDatabase(identifier.GetDatabaseName())) {
278+
if (!GlobalSystemTableLoader::IsSupported(identifier.GetTableName())) {
279+
return Status::NotExist(fmt::format("{} not exist", identifier.ToString()));
280+
}
281+
GlobalSystemTableContext context;
282+
context.catalog = const_cast<FileSystemCatalog*>(this);
283+
context.fs = fs_;
284+
context.warehouse = warehouse_;
285+
context.catalog_options = catalog_options_;
286+
PAIMON_ASSIGN_OR_RAISE(
287+
std::shared_ptr<SystemTable> system_table,
288+
GlobalSystemTableLoader::Load(identifier.GetTableName(), context));
289+
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> arrow_schema,
290+
system_table->ArrowSchema());
291+
return std::make_shared<SystemTableSchema>(std::move(arrow_schema));
292+
}
264293
PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable());
265294
if (is_system_table) {
266295
PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> system_table_name,

src/paimon/core/catalog/file_system_catalog.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ class Logger;
3838

3939
class FileSystemCatalog : public Catalog {
4040
public:
41-
FileSystemCatalog(const std::shared_ptr<FileSystem>& fs, const std::string& warehouse);
41+
FileSystemCatalog(const std::shared_ptr<FileSystem>& fs, const std::string& warehouse,
42+
const std::map<std::string, std::string>& catalog_options);
4243

4344
Status CreateDatabase(const std::string& db_name,
4445
const std::map<std::string, std::string>& options,
@@ -61,6 +62,7 @@ class FileSystemCatalog : public Catalog {
6162
Result<std::shared_ptr<Schema>> LoadTableSchema(const Identifier& identifier) const override;
6263
std::string GetRootPath() const override;
6364
std::shared_ptr<FileSystem> GetFileSystem() const override;
65+
const std::map<std::string, std::string>& GetOptions() const override;
6466
Result<std::shared_ptr<Table>> GetTable(const Identifier& identifier) const override;
6567
Result<std::vector<SnapshotInfo>> ListSnapshots(const Identifier& identifier,
6668
const std::string& branch) const override;
@@ -92,6 +94,7 @@ class FileSystemCatalog : public Catalog {
9294

9395
std::shared_ptr<FileSystem> fs_;
9496
std::string warehouse_;
97+
std::map<std::string, std::string> catalog_options_;
9598

9699
std::shared_ptr<Logger> logger_;
97100
};

0 commit comments

Comments
 (0)