Skip to content

Commit 00b71ad

Browse files
lishuxushuxu.li
lishuxu
and
shuxu.li
authored
feat: Refresh method support for table (#152)
Co-authored-by: shuxu.li <shuxu.li@clickzetta.com>
1 parent 7063f2b commit 00b71ad

File tree

5 files changed

+166
-3
lines changed

5 files changed

+166
-3
lines changed

‎src/iceberg/catalog/in_memory_catalog.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ class ICEBERG_EXPORT InMemoryCatalog
9090
const TableIdentifier& identifier,
9191
const std::string& metadata_file_location) override;
9292

93-
std::unique_ptr<Catalog::TableBuilder> BuildTable(const TableIdentifier& identifier,
94-
const Schema& schema) const override;
93+
std::unique_ptr<TableBuilder> BuildTable(const TableIdentifier& identifier,
94+
const Schema& schema) const override;
9595

9696
private:
9797
std::string catalog_name_;

‎src/iceberg/table.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,36 @@
2121

2222
#include <algorithm>
2323

24+
#include "iceberg/catalog.h"
2425
#include "iceberg/partition_spec.h"
2526
#include "iceberg/schema.h"
2627
#include "iceberg/sort_order.h"
2728
#include "iceberg/table_metadata.h"
2829
#include "iceberg/table_scan.h"
30+
#include "iceberg/util/macros.h"
2931

3032
namespace iceberg {
3133

3234
const std::string& Table::uuid() const { return metadata_->table_uuid; }
3335

36+
Status Table::Refresh() {
37+
if (!catalog_) {
38+
return NotSupported("Refresh is not supported for table without a catalog");
39+
}
40+
41+
ICEBERG_ASSIGN_OR_RAISE(auto refreshed_table, catalog_->LoadTable(identifier_));
42+
if (metadata_location_ != refreshed_table->metadata_location_) {
43+
metadata_ = std::move(refreshed_table->metadata_);
44+
metadata_location_ = std::move(refreshed_table->metadata_location_);
45+
io_ = std::move(refreshed_table->io_);
46+
47+
schemas_map_.reset();
48+
partition_spec_map_.reset();
49+
sort_orders_map_.reset();
50+
}
51+
return {};
52+
}
53+
3454
Result<std::shared_ptr<Schema>> Table::schema() const { return metadata_->Schema(); }
3555

3656
const std::shared_ptr<std::unordered_map<int32_t, std::shared_ptr<Schema>>>&

‎src/iceberg/table.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ class ICEBERG_EXPORT Table {
5757
/// \brief Returns the UUID of the table
5858
const std::string& uuid() const;
5959

60+
/// \brief Refresh the current table metadata
61+
Status Refresh();
62+
6063
/// \brief Return the schema for this table, return NotFoundError if not found
6164
Result<std::shared_ptr<Schema>> schema() const;
6265

@@ -116,7 +119,7 @@ class ICEBERG_EXPORT Table {
116119
private:
117120
const TableIdentifier identifier_;
118121
std::shared_ptr<TableMetadata> metadata_;
119-
const std::string metadata_location_;
122+
std::string metadata_location_;
120123
std::shared_ptr<FileIO> io_;
121124
std::shared_ptr<Catalog> catalog_;
122125

‎test/in_memory_catalog_test.cc‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@
2626
#include <gtest/gtest.h>
2727

2828
#include "iceberg/arrow/arrow_fs_file_io_internal.h"
29+
#include "iceberg/schema.h"
2930
#include "iceberg/table.h"
3031
#include "iceberg/table_metadata.h"
3132
#include "matchers.h"
33+
#include "mock_catalog.h"
3234
#include "test_common.h"
3335

3436
namespace iceberg {
@@ -115,6 +117,56 @@ TEST_F(InMemoryCatalogTest, RegisterTable) {
115117
ASSERT_EQ(table.value()->location(), "s3://bucket/test/location");
116118
}
117119

120+
TEST_F(InMemoryCatalogTest, RefreshTable) {
121+
TableIdentifier table_ident{.ns = {}, .name = "t1"};
122+
auto schema = std::make_shared<Schema>(
123+
std::vector<SchemaField>{SchemaField::MakeRequired(1, "x", int64())},
124+
/*schema_id=*/1);
125+
126+
std::shared_ptr<FileIO> io;
127+
128+
auto catalog = std::make_shared<MockCatalog>();
129+
// Mock 1st call to LoadTable
130+
EXPECT_CALL(*catalog, LoadTable(::testing::_))
131+
.WillOnce(::testing::Return(
132+
std::make_unique<Table>(table_ident,
133+
std::make_shared<TableMetadata>(TableMetadata{
134+
.schemas = {schema},
135+
.current_schema_id = 1,
136+
.current_snapshot_id = 1,
137+
.snapshots = {std::make_shared<Snapshot>(Snapshot{
138+
.snapshot_id = 1,
139+
.sequence_number = 1,
140+
})}}),
141+
"s3://location/1.json", io, catalog)));
142+
auto load_table_result = catalog->LoadTable(table_ident);
143+
ASSERT_THAT(load_table_result, IsOk());
144+
auto loaded_table = std::move(load_table_result.value());
145+
ASSERT_EQ(loaded_table->current_snapshot().value()->snapshot_id, 1);
146+
147+
// Mock 2nd call to LoadTable
148+
EXPECT_CALL(*catalog, LoadTable(::testing::_))
149+
.WillOnce(::testing::Return(
150+
std::make_unique<Table>(table_ident,
151+
std::make_shared<TableMetadata>(TableMetadata{
152+
.schemas = {schema},
153+
.current_schema_id = 1,
154+
.current_snapshot_id = 2,
155+
.snapshots = {std::make_shared<Snapshot>(Snapshot{
156+
.snapshot_id = 1,
157+
.sequence_number = 1,
158+
}),
159+
std::make_shared<Snapshot>(Snapshot{
160+
.snapshot_id = 2,
161+
.sequence_number = 2,
162+
})}}),
163+
"s3://location/2.json", io, catalog)));
164+
auto refreshed_result = loaded_table->Refresh();
165+
ASSERT_THAT(refreshed_result, IsOk());
166+
// check table is refreshed
167+
ASSERT_EQ(loaded_table->current_snapshot().value()->snapshot_id, 2);
168+
}
169+
118170
TEST_F(InMemoryCatalogTest, DropTable) {
119171
TableIdentifier tableIdent{.ns = {}, .name = "t1"};
120172
auto result = catalog_->DropTable(tableIdent, false);

‎test/mock_catalog.h‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <gmock/gmock.h>
23+
#include <gtest/gtest.h>
24+
25+
#include "iceberg/catalog.h"
26+
27+
namespace iceberg {
28+
29+
class MockCatalog : public Catalog {
30+
public:
31+
MockCatalog() = default;
32+
~MockCatalog() override = default;
33+
34+
MOCK_METHOD(std::string_view, name, (), (const, override));
35+
36+
MOCK_METHOD(Status, CreateNamespace,
37+
(const Namespace&, (const std::unordered_map<std::string, std::string>&)),
38+
(override));
39+
40+
MOCK_METHOD((Result<std::vector<Namespace>>), ListNamespaces, (const Namespace&),
41+
(const, override));
42+
43+
MOCK_METHOD((Result<std::unordered_map<std::string, std::string>>),
44+
GetNamespaceProperties, (const Namespace&), (const, override));
45+
46+
MOCK_METHOD(Status, UpdateNamespaceProperties,
47+
(const Namespace&, (const std::unordered_map<std::string, std::string>&),
48+
(const std::unordered_set<std::string>&)),
49+
(override));
50+
51+
MOCK_METHOD(Status, DropNamespace, (const Namespace&), (override));
52+
53+
MOCK_METHOD(Result<bool>, NamespaceExists, (const Namespace&), (const, override));
54+
55+
MOCK_METHOD((Result<std::vector<TableIdentifier>>), ListTables, (const Namespace&),
56+
(const, override));
57+
58+
MOCK_METHOD((Result<std::unique_ptr<Table>>), CreateTable,
59+
(const TableIdentifier&, const Schema&, const PartitionSpec&,
60+
const std::string&, (const std::unordered_map<std::string, std::string>&)),
61+
(override));
62+
63+
MOCK_METHOD((Result<std::unique_ptr<Table>>), UpdateTable,
64+
(const TableIdentifier&,
65+
(const std::vector<std::unique_ptr<UpdateRequirement>>&),
66+
(const std::vector<std::unique_ptr<MetadataUpdate>>&)),
67+
(override));
68+
69+
MOCK_METHOD((Result<std::shared_ptr<Transaction>>), StageCreateTable,
70+
(const TableIdentifier&, const Schema&, const PartitionSpec&,
71+
const std::string&, (const std::unordered_map<std::string, std::string>&)),
72+
(override));
73+
74+
MOCK_METHOD(Result<bool>, TableExists, (const TableIdentifier&), (const, override));
75+
76+
MOCK_METHOD(Status, DropTable, (const TableIdentifier&, bool), (override));
77+
78+
MOCK_METHOD((Result<std::unique_ptr<Table>>), LoadTable, (const TableIdentifier&),
79+
(override));
80+
81+
MOCK_METHOD((Result<std::shared_ptr<Table>>), RegisterTable,
82+
(const TableIdentifier&, const std::string&), (override));
83+
84+
MOCK_METHOD((std::unique_ptr<TableBuilder>), BuildTable,
85+
(const TableIdentifier&, const Schema&), (const, override));
86+
};
87+
88+
} // namespace iceberg

0 commit comments

Comments
 (0)