Skip to content

Commit 8055a60

Browse files
ebyhrebyhr
authored
REST: Add support for page-size in list_namespaces, list_tables, and list_views (#3377)
<!-- Thanks for opening a pull request! --> <!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. --> <!-- Closes #${GITHUB_ISSUE_ID} --> # Rationale for this change Follows REST catalog spec: * `/v1/{prefix}/views`: https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L1525-L1538 * `page-size`: https://github.com/apache/iceberg/blob/e7a5a87f26f9de5b200254155aa037368b13a29c/open-api/rest-catalog-open-api.yaml#L2029-L2038 ## Are these changes tested? Yes. ## Are there any user-facing changes? Add support for `page-size` in list_views in REST catalog. <!-- In the case of user-facing changes, please add the changelog label. -->
1 parent a24af23 commit 8055a60

2 files changed

Lines changed: 111 additions & 3 deletions

File tree

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,8 @@ class ScanPlanningMode(Enum):
267267
VIEW_ENDPOINTS_SUPPORTED = "view-endpoints-supported"
268268
VIEW_ENDPOINTS_SUPPORTED_DEFAULT = False
269269

270+
PAGE_SIZE = "rest-page-size"
271+
270272
NAMESPACE_SEPARATOR_PROPERTY = "namespace-separator"
271273
DEFAULT_NAMESPACE_SEPARATOR = b"\x1f".decode(UTF8)
272274

@@ -1042,11 +1044,17 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]:
10421044
namespace_concat = self._encode_namespace_path(namespace_tuple)
10431045
url = self.url(Endpoints.list_tables, namespace=namespace_concat)
10441046

1047+
params: dict[str, str] = {}
1048+
page_size = property_as_int(self.properties, PAGE_SIZE, None)
1049+
if page_size is not None:
1050+
if page_size <= 0:
1051+
raise ValueError(f"{PAGE_SIZE} must be a positive integer")
1052+
params["pageSize"] = str(page_size)
1053+
10451054
tables: list[Identifier] = []
10461055
page_token: str | None = None
10471056

10481057
while True:
1049-
params: dict[str, str] = {}
10501058
if page_token:
10511059
params["pageToken"] = page_token
10521060
response = self._session.get(url, params=params)
@@ -1150,11 +1158,20 @@ def list_views(self, namespace: str | Identifier) -> list[Identifier]:
11501158
namespace_concat = self._encode_namespace_path(namespace_tuple)
11511159
url = self.url(Endpoints.list_views, namespace=namespace_concat)
11521160

1161+
params: dict[str, str] = {}
1162+
page_size = property_as_int(self.properties, PAGE_SIZE, None)
1163+
if page_size is not None:
1164+
if page_size <= 0:
1165+
raise ValueError(f"{PAGE_SIZE} must be a positive integer")
1166+
params["pageSize"] = str(page_size)
1167+
11531168
views: list[Identifier] = []
11541169
page_token: str | None = None
11551170

11561171
while True:
1157-
params = {"pageToken": page_token} if page_token else None
1172+
if page_token:
1173+
params["pageToken"] = page_token
1174+
11581175
response = self._session.get(url, params=params)
11591176
try:
11601177
response.raise_for_status()
@@ -1263,11 +1280,17 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]:
12631280
self._check_endpoint(Capability.V1_LIST_NAMESPACES)
12641281
namespace_tuple = self.identifier_to_tuple(namespace)
12651282

1283+
params: dict[str, str] = {}
1284+
page_size = property_as_int(self.properties, PAGE_SIZE, None)
1285+
if page_size is not None:
1286+
if page_size <= 0:
1287+
raise ValueError(f"{PAGE_SIZE} must be a positive integer")
1288+
params["pageSize"] = str(page_size)
1289+
12661290
namespaces: list[Identifier] = []
12671291
page_token: str | None = None
12681292

12691293
while True:
1270-
params: dict[str, str] = {}
12711294
if namespace_tuple:
12721295
params["parent"] = self._encode_namespace_path(namespace_tuple)
12731296
if page_token:

‎tests/catalog/test_rest.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
DEFAULT_ENDPOINTS,
3636
EMPTY_BODY_SHA256,
3737
OAUTH2_SERVER_URI,
38+
PAGE_SIZE,
3839
SIGV4_MAX_RETRIES,
3940
SIGV4_MAX_RETRIES_DEFAULT,
4041
SNAPSHOT_LOADING_MODE,
@@ -564,6 +565,29 @@ def test_list_tables_paginated_200_none_next_page_token(rest_mock: Mocker) -> No
564565
]
565566

566567

568+
def test_list_tables_page_size(rest_mock: Mocker) -> None:
569+
namespace = "examples"
570+
rest_mock.get(
571+
f"{TEST_URI}v1/namespaces/{namespace}/tables",
572+
json={
573+
"identifiers": [
574+
{"namespace": ["examples"], "name": "table1"},
575+
{"namespace": ["examples"], "name": "table2"},
576+
],
577+
},
578+
status_code=200,
579+
request_headers=TEST_HEADERS,
580+
)
581+
582+
result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE: "100"}).list_tables(namespace)
583+
assert rest_mock.last_request.url == f"{TEST_URI}v1/namespaces/examples/tables?pageSize=100"
584+
585+
assert result == [
586+
("examples", "table1"),
587+
("examples", "table2"),
588+
]
589+
590+
567591
def test_list_tables_200_sigv4(rest_mock: Mocker) -> None:
568592
namespace = "examples"
569593
rest_mock.get(
@@ -810,6 +834,48 @@ def test_list_views_paginated_200_none_next_page_token(rest_mock: Mocker) -> Non
810834
]
811835

812836

837+
def test_list_views_page_size(rest_mock: Mocker) -> None:
838+
namespace = "examples"
839+
rest_mock.get(
840+
f"{TEST_URI}v1/namespaces/{namespace}/views",
841+
json={
842+
"identifiers": [
843+
{"namespace": ["examples"], "name": "view1"},
844+
{"namespace": ["examples"], "name": "view2"},
845+
],
846+
},
847+
status_code=200,
848+
request_headers=TEST_HEADERS,
849+
)
850+
851+
result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE: "100"}).list_views(namespace)
852+
assert rest_mock.last_request.url == f"{TEST_URI}v1/namespaces/examples/views?pageSize=100"
853+
854+
assert result == [
855+
("examples", "view1"),
856+
("examples", "view2"),
857+
]
858+
859+
860+
def test_list_views_invalid_page_size(rest_mock: Mocker) -> None:
861+
namespace = "examples"
862+
rest_mock.get(
863+
f"{TEST_URI}v1/namespaces/{namespace}/views",
864+
json={
865+
"identifiers": [
866+
{"namespace": ["examples"], "name": "view1"},
867+
{"namespace": ["examples"], "name": "view2"},
868+
],
869+
},
870+
status_code=200,
871+
request_headers=TEST_HEADERS,
872+
)
873+
874+
with pytest.raises(ValueError) as e:
875+
RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE: "0"}).list_views(namespace)
876+
assert str(e.value) == "rest-page-size must be a positive integer"
877+
878+
813879
def test_list_views_200_sigv4(rest_mock: Mocker) -> None:
814880
namespace = "examples"
815881
rest_mock.get(
@@ -1006,6 +1072,25 @@ def test_list_namespaces_paginated_200_none_next_page_token(rest_mock: Mocker) -
10061072
]
10071073

10081074

1075+
def test_list_namespaces_page_size(rest_mock: Mocker) -> None:
1076+
rest_mock.get(
1077+
f"{TEST_URI}v1/namespaces",
1078+
json={
1079+
"namespaces": [["ns1"], ["ns2"]],
1080+
},
1081+
status_code=200,
1082+
request_headers=TEST_HEADERS,
1083+
)
1084+
1085+
result = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{PAGE_SIZE: "100"}).list_namespaces()
1086+
assert rest_mock.last_request.url == f"{TEST_URI}v1/namespaces?pageSize=100"
1087+
1088+
assert result == [
1089+
("ns1",),
1090+
("ns2",),
1091+
]
1092+
1093+
10091094
def test_list_namespace_with_parent_404(rest_mock: Mocker) -> None:
10101095
rest_mock.get(
10111096
f"{TEST_URI}v1/namespaces?parent=some_namespace",

0 commit comments

Comments
 (0)