Skip to content

Commit 96ba8de

Browse files
kevinjqliukevinjqliu
committed
create_table write metadata file
1 parent 67c028a commit 96ba8de

File tree

4 files changed

+28
-69
lines changed

4 files changed

+28
-69
lines changed

‎pyiceberg/catalog/in_memory.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
NoSuchTableError,
3939
TableAlreadyExistsError,
4040
)
41-
from pyiceberg.io import WAREHOUSE
41+
from pyiceberg.io import WAREHOUSE, load_file_io
4242
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec
4343
from pyiceberg.schema import Schema
4444
from pyiceberg.table import (
@@ -94,8 +94,7 @@ def create_table(
9494
if not location:
9595
location = f'{self._warehouse_location}/{"/".join(identifier)}'
9696

97-
metadata_location = f'{self._warehouse_location}/{"/".join(identifier)}/metadata/metadata.json'
98-
97+
metadata_location = self._get_metadata_location(location=location)
9998
metadata = new_table_metadata(
10099
schema=schema,
101100
partition_spec=partition_spec,
@@ -104,11 +103,14 @@ def create_table(
104103
properties=properties,
105104
table_uuid=table_uuid,
106105
)
106+
io = load_file_io({**self.properties, **properties}, location=location)
107+
self._write_metadata(metadata, io, metadata_location)
108+
107109
table = Table(
108110
identifier=identifier,
109111
metadata=metadata,
110112
metadata_location=metadata_location,
111-
io=self._load_file_io(properties=metadata.properties, location=metadata_location),
113+
io=io,
112114
catalog=self,
113115
)
114116
self.__tables[identifier] = table

‎pyiceberg/cli/output.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ def describe_properties(self, properties: Properties) -> None:
157157
Console().print(output_table)
158158

159159
def text(self, response: str) -> None:
160-
Console().print(response)
160+
Console(soft_wrap=True).print(response)
161161

162162
def schema(self, schema: Schema) -> None:
163163
output_table = self._table

‎tests/catalog/test_base.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ def catalog(tmp_path: PosixPath) -> InMemoryCatalog:
6464
NestedField(2, "y", LongType(), doc="comment"),
6565
NestedField(3, "z", LongType()),
6666
)
67-
TEST_TABLE_LOCATION = "protocol://some/location"
6867
TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000))
6968
TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"}
7069
NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)"
@@ -123,13 +122,25 @@ def test_create_table(catalog: InMemoryCatalog) -> None:
123122
table = catalog.create_table(
124123
identifier=TEST_TABLE_IDENTIFIER,
125124
schema=TEST_TABLE_SCHEMA,
126-
location=TEST_TABLE_LOCATION,
127125
partition_spec=TEST_TABLE_PARTITION_SPEC,
128126
properties=TEST_TABLE_PROPERTIES,
129127
)
130128
assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table
131129

132130

131+
def test_create_table_location_override(catalog: InMemoryCatalog) -> None:
132+
new_location = f"{catalog._warehouse_location}/new_location"
133+
table = catalog.create_table(
134+
identifier=TEST_TABLE_IDENTIFIER,
135+
schema=TEST_TABLE_SCHEMA,
136+
location=new_location,
137+
partition_spec=TEST_TABLE_PARTITION_SPEC,
138+
properties=TEST_TABLE_PROPERTIES,
139+
)
140+
assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table
141+
assert table.location() == new_location
142+
143+
133144
@pytest.mark.parametrize(
134145
"schema,expected",
135146
[
@@ -151,7 +162,6 @@ def test_create_table_pyarrow_schema(catalog: InMemoryCatalog, pyarrow_schema_si
151162
table = catalog.create_table(
152163
identifier=TEST_TABLE_IDENTIFIER,
153164
schema=pyarrow_schema_simple_without_ids,
154-
location=TEST_TABLE_LOCATION,
155165
properties=TEST_TABLE_PROPERTIES,
156166
)
157167
assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table

‎tests/cli/test_console.py‎

Lines changed: 8 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,15 @@
1717
import datetime
1818
import os
1919
import uuid
20+
from pathlib import PosixPath
2021
from unittest.mock import MagicMock
2122

2223
import pytest
2324
from click.testing import CliRunner
2425
from pytest_mock import MockFixture
2526

26-
from pyiceberg.catalog.in_memory import DEFAULT_WAREHOUSE_LOCATION
2727
from pyiceberg.cli.console import run
28+
from pyiceberg.io import WAREHOUSE
2829
from pyiceberg.partitioning import PartitionField, PartitionSpec
2930
from pyiceberg.schema import Schema
3031
from pyiceberg.transforms import IdentityTransform
@@ -52,8 +53,10 @@ def env_vars(mocker: MockFixture) -> None:
5253

5354

5455
@pytest.fixture(name="catalog")
55-
def fixture_catalog(mocker: MockFixture) -> InMemoryCatalog:
56-
in_memory_catalog = InMemoryCatalog("test.in_memory.catalog", **{"test.key": "test.value"})
56+
def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog:
57+
in_memory_catalog = InMemoryCatalog(
58+
"test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}
59+
)
5760
mocker.patch("pyiceberg.cli.console.load_catalog", return_value=in_memory_catalog)
5861
return in_memory_catalog
5962

@@ -79,7 +82,6 @@ def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None:
7982
NestedField(2, "y", LongType(), doc="comment"),
8083
NestedField(3, "z", LongType()),
8184
)
82-
TEST_TABLE_LOCATION = "s3://bucket/test/location"
8385
TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000))
8486
TEST_TABLE_PROPERTIES = {"read.split.target.size": "134217728"}
8587
TEST_TABLE_UUID = uuid.UUID("d20125c8-7284-442c-9aea-15fee620737c")
@@ -101,7 +103,6 @@ def test_list_namespace(catalog: InMemoryCatalog) -> None:
101103
catalog.create_table(
102104
identifier=TEST_TABLE_IDENTIFIER,
103105
schema=TEST_TABLE_SCHEMA,
104-
location=TEST_TABLE_LOCATION,
105106
partition_spec=TEST_TABLE_PARTITION_SPEC,
106107
properties=TEST_TABLE_PROPERTIES,
107108
)
@@ -138,7 +139,6 @@ def test_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> No
138139
catalog.create_table(
139140
identifier=TEST_TABLE_IDENTIFIER,
140141
schema=TEST_TABLE_SCHEMA,
141-
location=TEST_TABLE_LOCATION,
142142
partition_spec=TEST_TABLE_PARTITION_SPEC,
143143
table_uuid=TEST_TABLE_UUID,
144144
)
@@ -182,7 +182,6 @@ def test_schema(catalog: InMemoryCatalog) -> None:
182182
catalog.create_table(
183183
identifier=TEST_TABLE_IDENTIFIER,
184184
schema=TEST_TABLE_SCHEMA,
185-
location=TEST_TABLE_LOCATION,
186185
partition_spec=TEST_TABLE_PARTITION_SPEC,
187186
)
188187

@@ -211,7 +210,6 @@ def test_spec(catalog: InMemoryCatalog) -> None:
211210
catalog.create_table(
212211
identifier=TEST_TABLE_IDENTIFIER,
213212
schema=TEST_TABLE_SCHEMA,
214-
location=TEST_TABLE_LOCATION,
215213
partition_spec=TEST_TABLE_PARTITION_SPEC,
216214
)
217215

@@ -240,7 +238,6 @@ def test_uuid(catalog: InMemoryCatalog) -> None:
240238
catalog.create_table(
241239
identifier=TEST_TABLE_IDENTIFIER,
242240
schema=TEST_TABLE_SCHEMA,
243-
location=TEST_TABLE_LOCATION,
244241
partition_spec=TEST_TABLE_PARTITION_SPEC,
245242
table_uuid=TEST_TABLE_UUID,
246243
)
@@ -266,25 +263,10 @@ def test_location(catalog: InMemoryCatalog) -> None:
266263
schema=TEST_TABLE_SCHEMA,
267264
partition_spec=TEST_TABLE_PARTITION_SPEC,
268265
)
269-
270-
runner = CliRunner()
271-
result = runner.invoke(run, ["location", "default.my_table"])
272-
assert result.exit_code == 0
273-
assert result.output == f"""{DEFAULT_WAREHOUSE_LOCATION}/default/my_table\n"""
274-
275-
276-
def test_location_override(catalog: InMemoryCatalog) -> None:
277-
catalog.create_table(
278-
identifier=TEST_TABLE_IDENTIFIER,
279-
schema=TEST_TABLE_SCHEMA,
280-
location=TEST_TABLE_LOCATION,
281-
partition_spec=TEST_TABLE_PARTITION_SPEC,
282-
)
283-
284266
runner = CliRunner()
285267
result = runner.invoke(run, ["location", "default.my_table"])
286268
assert result.exit_code == 0
287-
assert result.output == f"""{TEST_TABLE_LOCATION}\n"""
269+
assert result.output == f"""{catalog._warehouse_location}/default/my_table\n"""
288270

289271

290272
def test_location_does_not_exists(catalog: InMemoryCatalog) -> None:
@@ -300,7 +282,6 @@ def test_drop_table(catalog: InMemoryCatalog) -> None:
300282
catalog.create_table(
301283
identifier=TEST_TABLE_IDENTIFIER,
302284
schema=TEST_TABLE_SCHEMA,
303-
location=TEST_TABLE_LOCATION,
304285
partition_spec=TEST_TABLE_PARTITION_SPEC,
305286
)
306287

@@ -341,7 +322,6 @@ def test_rename_table(catalog: InMemoryCatalog) -> None:
341322
catalog.create_table(
342323
identifier=TEST_TABLE_IDENTIFIER,
343324
schema=TEST_TABLE_SCHEMA,
344-
location=TEST_TABLE_LOCATION,
345325
partition_spec=TEST_TABLE_PARTITION_SPEC,
346326
)
347327

@@ -364,7 +344,6 @@ def test_properties_get_table(catalog: InMemoryCatalog) -> None:
364344
catalog.create_table(
365345
identifier=TEST_TABLE_IDENTIFIER,
366346
schema=TEST_TABLE_SCHEMA,
367-
location=TEST_TABLE_LOCATION,
368347
partition_spec=TEST_TABLE_PARTITION_SPEC,
369348
properties=TEST_TABLE_PROPERTIES,
370349
)
@@ -379,7 +358,6 @@ def test_properties_get_table_specific_property(catalog: InMemoryCatalog) -> Non
379358
catalog.create_table(
380359
identifier=TEST_TABLE_IDENTIFIER,
381360
schema=TEST_TABLE_SCHEMA,
382-
location=TEST_TABLE_LOCATION,
383361
partition_spec=TEST_TABLE_PARTITION_SPEC,
384362
properties=TEST_TABLE_PROPERTIES,
385363
)
@@ -394,7 +372,6 @@ def test_properties_get_table_specific_property_that_doesnt_exist(catalog: InMem
394372
catalog.create_table(
395373
identifier=TEST_TABLE_IDENTIFIER,
396374
schema=TEST_TABLE_SCHEMA,
397-
location=TEST_TABLE_LOCATION,
398375
partition_spec=TEST_TABLE_PARTITION_SPEC,
399376
properties=TEST_TABLE_PROPERTIES,
400377
)
@@ -463,7 +440,6 @@ def test_properties_set_table(catalog: InMemoryCatalog) -> None:
463440
catalog.create_table(
464441
identifier=TEST_TABLE_IDENTIFIER,
465442
schema=TEST_TABLE_SCHEMA,
466-
location=TEST_TABLE_LOCATION,
467443
partition_spec=TEST_TABLE_PARTITION_SPEC,
468444
)
469445

@@ -504,7 +480,6 @@ def test_properties_remove_table(catalog: InMemoryCatalog) -> None:
504480
catalog.create_table(
505481
identifier=TEST_TABLE_IDENTIFIER,
506482
schema=TEST_TABLE_SCHEMA,
507-
location=TEST_TABLE_LOCATION,
508483
partition_spec=TEST_TABLE_PARTITION_SPEC,
509484
properties=TEST_TABLE_PROPERTIES,
510485
)
@@ -519,7 +494,6 @@ def test_properties_remove_table_property_does_not_exists(catalog: InMemoryCatal
519494
catalog.create_table(
520495
identifier=TEST_TABLE_IDENTIFIER,
521496
schema=TEST_TABLE_SCHEMA,
522-
location=TEST_TABLE_LOCATION,
523497
partition_spec=TEST_TABLE_PARTITION_SPEC,
524498
)
525499

@@ -551,7 +525,6 @@ def test_json_list_namespace(catalog: InMemoryCatalog) -> None:
551525
catalog.create_table(
552526
identifier=TEST_TABLE_IDENTIFIER,
553527
schema=TEST_TABLE_SCHEMA,
554-
location=TEST_TABLE_LOCATION,
555528
partition_spec=TEST_TABLE_PARTITION_SPEC,
556529
)
557530

@@ -584,7 +557,6 @@ def test_json_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None)
584557
catalog.create_table(
585558
identifier=TEST_TABLE_IDENTIFIER,
586559
schema=TEST_TABLE_SCHEMA,
587-
location=TEST_TABLE_LOCATION,
588560
partition_spec=TEST_TABLE_PARTITION_SPEC,
589561
table_uuid=TEST_TABLE_UUID,
590562
)
@@ -614,7 +586,6 @@ def test_json_schema(catalog: InMemoryCatalog) -> None:
614586
catalog.create_table(
615587
identifier=TEST_TABLE_IDENTIFIER,
616588
schema=TEST_TABLE_SCHEMA,
617-
location=TEST_TABLE_LOCATION,
618589
partition_spec=TEST_TABLE_PARTITION_SPEC,
619590
)
620591

@@ -640,7 +611,6 @@ def test_json_spec(catalog: InMemoryCatalog) -> None:
640611
catalog.create_table(
641612
identifier=TEST_TABLE_IDENTIFIER,
642613
schema=TEST_TABLE_SCHEMA,
643-
location=TEST_TABLE_LOCATION,
644614
partition_spec=TEST_TABLE_PARTITION_SPEC,
645615
)
646616

@@ -663,7 +633,6 @@ def test_json_uuid(catalog: InMemoryCatalog) -> None:
663633
catalog.create_table(
664634
identifier=TEST_TABLE_IDENTIFIER,
665635
schema=TEST_TABLE_SCHEMA,
666-
location=TEST_TABLE_LOCATION,
667636
partition_spec=TEST_TABLE_PARTITION_SPEC,
668637
table_uuid=TEST_TABLE_UUID,
669638
)
@@ -693,21 +662,7 @@ def test_json_location(catalog: InMemoryCatalog) -> None:
693662
runner = CliRunner()
694663
result = runner.invoke(run, ["--output=json", "location", "default.my_table"])
695664
assert result.exit_code == 0
696-
assert result.output == f'"{DEFAULT_WAREHOUSE_LOCATION}/default/my_table"\n'
697-
698-
699-
def test_json_location_override(catalog: InMemoryCatalog) -> None:
700-
catalog.create_table(
701-
identifier=TEST_TABLE_IDENTIFIER,
702-
schema=TEST_TABLE_SCHEMA,
703-
location=TEST_TABLE_LOCATION,
704-
partition_spec=TEST_TABLE_PARTITION_SPEC,
705-
)
706-
707-
runner = CliRunner()
708-
result = runner.invoke(run, ["--output=json", "location", "default.my_table"])
709-
assert result.exit_code == 0
710-
assert result.output == f'"{TEST_TABLE_LOCATION}"\n'
665+
assert result.output == f'"{catalog._warehouse_location}/default/my_table"\n'
711666

712667

713668
def test_json_location_does_not_exists(catalog: InMemoryCatalog) -> None:
@@ -723,7 +678,6 @@ def test_json_drop_table(catalog: InMemoryCatalog) -> None:
723678
catalog.create_table(
724679
identifier=TEST_TABLE_IDENTIFIER,
725680
schema=TEST_TABLE_SCHEMA,
726-
location=TEST_TABLE_LOCATION,
727681
partition_spec=TEST_TABLE_PARTITION_SPEC,
728682
)
729683

@@ -764,7 +718,6 @@ def test_json_rename_table(catalog: InMemoryCatalog) -> None:
764718
catalog.create_table(
765719
identifier=TEST_TABLE_IDENTIFIER,
766720
schema=TEST_TABLE_SCHEMA,
767-
location=TEST_TABLE_LOCATION,
768721
partition_spec=TEST_TABLE_PARTITION_SPEC,
769722
)
770723

@@ -787,7 +740,6 @@ def test_json_properties_get_table(catalog: InMemoryCatalog) -> None:
787740
catalog.create_table(
788741
identifier=TEST_TABLE_IDENTIFIER,
789742
schema=TEST_TABLE_SCHEMA,
790-
location=TEST_TABLE_LOCATION,
791743
partition_spec=TEST_TABLE_PARTITION_SPEC,
792744
properties=TEST_TABLE_PROPERTIES,
793745
)
@@ -802,7 +754,6 @@ def test_json_properties_get_table_specific_property(catalog: InMemoryCatalog) -
802754
catalog.create_table(
803755
identifier=TEST_TABLE_IDENTIFIER,
804756
schema=TEST_TABLE_SCHEMA,
805-
location=TEST_TABLE_LOCATION,
806757
partition_spec=TEST_TABLE_PARTITION_SPEC,
807758
properties=TEST_TABLE_PROPERTIES,
808759
)
@@ -817,7 +768,6 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog:
817768
catalog.create_table(
818769
identifier=TEST_TABLE_IDENTIFIER,
819770
schema=TEST_TABLE_SCHEMA,
820-
location=TEST_TABLE_LOCATION,
821771
partition_spec=TEST_TABLE_PARTITION_SPEC,
822772
properties=TEST_TABLE_PROPERTIES,
823773
)
@@ -891,7 +841,6 @@ def test_json_properties_set_table(catalog: InMemoryCatalog) -> None:
891841
catalog.create_table(
892842
identifier=TEST_TABLE_IDENTIFIER,
893843
schema=TEST_TABLE_SCHEMA,
894-
location=TEST_TABLE_LOCATION,
895844
partition_spec=TEST_TABLE_PARTITION_SPEC,
896845
properties=TEST_TABLE_PROPERTIES,
897846
)
@@ -937,7 +886,6 @@ def test_json_properties_remove_table(catalog: InMemoryCatalog) -> None:
937886
catalog.create_table(
938887
identifier=TEST_TABLE_IDENTIFIER,
939888
schema=TEST_TABLE_SCHEMA,
940-
location=TEST_TABLE_LOCATION,
941889
partition_spec=TEST_TABLE_PARTITION_SPEC,
942890
properties=TEST_TABLE_PROPERTIES,
943891
)
@@ -952,7 +900,6 @@ def test_json_properties_remove_table_property_does_not_exists(catalog: InMemory
952900
catalog.create_table(
953901
identifier=TEST_TABLE_IDENTIFIER,
954902
schema=TEST_TABLE_SCHEMA,
955-
location=TEST_TABLE_LOCATION,
956903
partition_spec=TEST_TABLE_PARTITION_SPEC,
957904
properties=TEST_TABLE_PROPERTIES,
958905
)

0 commit comments

Comments
 (0)