Skip to content

Commit 5991bbd

Browse files
rambleraptorrambleraptor
committed
Wire up Polaris to CI
1 parent 4173ef7 commit 5991bbd

File tree

5 files changed

+306
-0
lines changed

5 files changed

+306
-0
lines changed

‎.github/workflows/python-ci-polaris.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: "Python CI - Polaris"
19+
20+
on:
21+
push:
22+
branches:
23+
- 'main'
24+
pull_request:
25+
paths:
26+
- 'pyiceberg/**'
27+
- 'tests/**'
28+
- 'dev/docker-compose-polaris.yml'
29+
- 'dev/provision_polaris.py'
30+
- '.github/workflows/python-ci-polaris.yml'
31+
- 'Makefile'
32+
- 'pyproject.toml'
33+
- 'uv.lock'
34+
35+
concurrency:
36+
group: ${{ github.workflow }}-${{ github.ref }}
37+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
38+
39+
jobs:
40+
polaris-integration-test:
41+
runs-on: ubuntu-latest
42+
steps:
43+
- uses: actions/checkout@v6
44+
- uses: actions/setup-python@v6
45+
with:
46+
python-version: '3.12'
47+
- name: Install UV
48+
uses: astral-sh/setup-uv@v7
49+
with:
50+
enable-cache: true
51+
- name: Install
52+
run: make install
53+
- name: Run Polaris integration tests
54+
run: make test-polaris
55+
- name: Show debug logs
56+
if: ${{ failure() }}
57+
run: docker compose -f dev/docker-compose-polaris.yml logs

‎Makefile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ test: ## Run all unit tests (excluding integration)
108108

109109
test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests
110110

111+
test-polaris: test-polaris-setup test-polaris-exec test-polaris-cleanup ## Run Polaris integration tests
112+
111113
test-integration-setup: install ## Start Docker services for integration tests
112114
docker compose -f dev/docker-compose-integration.yml kill
113115
docker compose -f dev/docker-compose-integration.yml rm -f
@@ -123,6 +125,38 @@ test-integration-cleanup: ## Clean up integration test environment
123125
fi
124126
$(CLEANUP_COMMAND)
125127

128+
test-polaris-setup: install ## Start Docker services for Polaris integration tests
129+
docker compose -f dev/docker-compose-polaris.yml kill
130+
docker compose -f dev/docker-compose-polaris.yml rm -f
131+
docker compose -f dev/docker-compose-polaris.yml up -d --build --wait
132+
uv run $(PYTHON_ARG) python dev/provision_polaris.py > dev/polaris_creds.env
133+
134+
test-polaris-exec: ## Run Polaris integration tests
135+
@eval $$(cat dev/polaris_creds.env) && \
136+
PYICEBERG_TEST_CATALOG="polaris" \
137+
PYICEBERG_CATALOG__POLARIS__TYPE="rest" \
138+
PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" \
139+
PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" \
140+
PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$$CLIENT_ID:$$CLIENT_SECRET" \
141+
PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" \
142+
PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" \
143+
PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" \
144+
PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" \
145+
PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" \
146+
PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="admin" \
147+
PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="password" \
148+
PYICEBERG_CATALOG__POLARIS__S3__REGION="us-east-1" \
149+
$(TEST_RUNNER) pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" $(PYTEST_ARGS)
150+
# Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request.
151+
152+
153+
test-polaris-cleanup: ## Clean up Polaris integration test environment
154+
@if [ "${KEEP_COMPOSE}" != "1" ]; then \
155+
echo "Cleaning up Polaris Docker containers..."; \
156+
docker compose -f dev/docker-compose-polaris.yml down -v --remove-orphans --timeout 0 2>/dev/null || true; \
157+
rm -f dev/polaris_creds.env; \
158+
fi
159+
126160
test-integration-rebuild: ## Rebuild integration Docker services from scratch
127161
docker compose -f dev/docker-compose-integration.yml kill
128162
docker compose -f dev/docker-compose-integration.yml rm -f

‎dev/docker-compose-polaris.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
services:
19+
polaris:
20+
image: apache/polaris:latest
21+
container_name: pyiceberg-polaris
22+
networks:
23+
iceberg_net:
24+
ports:
25+
- 8181:8181
26+
- 8182:8182
27+
environment:
28+
- POLARIS_BOOTSTRAP_CREDENTIALS=POLARIS,root,s3cr3t
29+
- polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true
30+
- polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3"]
31+
- polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true
32+
- polaris.readiness.ignore-severe-issues=true
33+
- AWS_ACCESS_KEY_ID=admin
34+
- AWS_SECRET_ACCESS_KEY=password
35+
- AWS_REGION=us-east-1
36+
healthcheck:
37+
test: ["CMD", "curl", "http://localhost:8182/q/health"]
38+
interval: 10s
39+
timeout: 10s
40+
retries: 5
41+
minio:
42+
image: minio/minio
43+
container_name: pyiceberg-polaris-minio
44+
networks:
45+
iceberg_net:
46+
aliases:
47+
- warehouse.minio
48+
ports:
49+
- 9001:9001
50+
- 9000:9000
51+
environment:
52+
- MINIO_ROOT_USER=admin
53+
- MINIO_ROOT_PASSWORD=password
54+
- MINIO_DOMAIN=minio
55+
command: ["server", "/data", "--console-address", ":9001"]
56+
mc:
57+
image: minio/mc
58+
container_name: pyiceberg-polaris-mc
59+
networks:
60+
iceberg_net:
61+
depends_on:
62+
- minio
63+
environment:
64+
- AWS_ACCESS_KEY_ID=admin
65+
- AWS_SECRET_ACCESS_KEY=password
66+
- AWS_REGION=us-east-1
67+
entrypoint: >
68+
/bin/sh -c "
69+
until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
70+
/usr/bin/mc mb minio/warehouse;
71+
/usr/bin/mc policy set public minio/warehouse;
72+
tail -f /dev/null
73+
"
74+
75+
networks:
76+
iceberg_net:

‎dev/polaris_creds.env‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
CLIENT_ID=dae157528c18583c
2+
CLIENT_SECRET=e056dc0ab9f23c2ab87d9da4ad93cdaa

‎dev/provision_polaris.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
19+
import requests
20+
21+
POLARIS_URL = "http://localhost:8181/api/management/v1"
22+
POLARIS_TOKEN_URL = "http://localhost:8181/api/catalog/v1/oauth/tokens"
23+
24+
25+
def get_token(client_id: str, client_secret: str) -> str:
26+
response = requests.post(
27+
POLARIS_TOKEN_URL,
28+
data={
29+
"grant_type": "client_credentials",
30+
"client_id": client_id,
31+
"client_secret": client_secret,
32+
"scope": "PRINCIPAL_ROLE:ALL",
33+
},
34+
headers={"realm": "POLARIS"},
35+
)
36+
response.raise_for_status()
37+
return response.json()["access_token"]
38+
39+
40+
def provision() -> None:
41+
# Initial authentication with root credentials
42+
token = get_token("root", "s3cr3t")
43+
headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", "realm": "POLARIS"}
44+
45+
# 1. Create Principal
46+
principal_name = "pyiceberg_principal"
47+
principal_resp = requests.post(
48+
f"{POLARIS_URL}/principals",
49+
headers=headers,
50+
json={"name": principal_name, "type": "PRINCIPAL"},
51+
)
52+
if principal_resp.status_code == 409:
53+
principal_resp = requests.post(
54+
f"{POLARIS_URL}/principals/{principal_name}/rotate-credentials",
55+
headers=headers,
56+
)
57+
principal_resp.raise_for_status()
58+
principal_data = principal_resp.json()
59+
client_id = principal_data["credentials"]["clientId"]
60+
client_secret = principal_data["credentials"]["clientSecret"]
61+
62+
# 2. Assign service_admin role to our principal
63+
requests.put(
64+
f"{POLARIS_URL}/principals/{principal_name}/principal-roles",
65+
headers=headers,
66+
json={"principalRole": {"name": "service_admin"}},
67+
).raise_for_status()
68+
69+
# 3. Create Principal Role for catalog access
70+
role_name = "pyiceberg_role"
71+
requests.post(
72+
f"{POLARIS_URL}/principal-roles",
73+
headers=headers,
74+
json={"principalRole": {"name": role_name}},
75+
) # Ignore error if exists
76+
77+
# 4. Link Principal to Principal Role
78+
requests.put(
79+
f"{POLARIS_URL}/principals/{principal_name}/principal-roles",
80+
headers=headers,
81+
json={"principalRole": {"name": role_name}},
82+
).raise_for_status()
83+
84+
# 5. Create Catalog
85+
catalog_name = "polaris"
86+
requests.post(
87+
f"{POLARIS_URL}/catalogs",
88+
headers=headers,
89+
json={
90+
"catalog": {
91+
"name": catalog_name,
92+
"type": "INTERNAL",
93+
"readOnly": False,
94+
"properties": {
95+
"default-base-location": "s3://warehouse/polaris/",
96+
"polaris.config.drop-with-purge.enabled": "true",
97+
},
98+
"storageConfigInfo": {
99+
"storageType": "S3",
100+
"allowedLocations": ["s3://warehouse/polaris/"],
101+
"region": "us-east-1",
102+
"endpoint": "http://minio:9000",
103+
},
104+
}
105+
},
106+
) # Ignore error if exists
107+
108+
# 6. Link catalog_admin role to our principal role
109+
requests.put(
110+
f"{POLARIS_URL}/principal-roles/{role_name}/catalog-roles/{catalog_name}",
111+
headers=headers,
112+
json={"catalogRole": {"name": "catalog_admin"}},
113+
).raise_for_status()
114+
115+
# 7. Grant explicit privileges to catalog_admin role for this catalog
116+
for privilege in [
117+
"CATALOG_MANAGE_CONTENT",
118+
"CATALOG_MANAGE_METADATA",
119+
"TABLE_CREATE",
120+
"TABLE_WRITE_DATA",
121+
"TABLE_LIST",
122+
"NAMESPACE_CREATE",
123+
"NAMESPACE_LIST",
124+
]:
125+
requests.put(
126+
f"{POLARIS_URL}/catalogs/{catalog_name}/catalog-roles/catalog_admin/grants",
127+
headers=headers,
128+
json={"grant": {"type": "catalog", "privilege": privilege}},
129+
).raise_for_status()
130+
131+
# Print credentials for use in CI
132+
print(f"CLIENT_ID={client_id}")
133+
print(f"CLIENT_SECRET={client_secret}")
134+
135+
136+
if __name__ == "__main__":
137+
provision()

0 commit comments

Comments
 (0)