Make docker-compose-polaris match the quick start

Author: rambleraptor

Makefile

@@ -129,26 +129,10 @@ test-polaris-setup: install ## Start Docker services for Polaris integration tes
 	docker compose -f dev/docker-compose-polaris.yml kill
 	docker compose -f dev/docker-compose-polaris.yml rm -f
 	docker compose -f dev/docker-compose-polaris.yml up -d --build --wait
-	uv run $(PYTHON_ARG) python dev/provision_polaris.py > dev/polaris_creds.env
+	docker compose -f dev/docker-compose-polaris.yml exec -T polaris-setup cat /tmp/polaris_creds.env > dev/polaris_creds.env
 
 test-polaris-exec: ## Run Polaris integration tests
-	@eval $$(cat dev/polaris_creds.env) && \
-	PYICEBERG_TEST_CATALOG="polaris" \
-	PYICEBERG_CATALOG__POLARIS__TYPE="rest" \
-	PYICEBERG_CATALOG__POLARIS__URI="http://localhost:8181/api/catalog" \
-	PYICEBERG_CATALOG__POLARIS__OAUTH2_SERVER_URI="http://localhost:8181/api/catalog/v1/oauth/tokens" \
-	PYICEBERG_CATALOG__POLARIS__CREDENTIAL="$$CLIENT_ID:$$CLIENT_SECRET" \
-	PYICEBERG_CATALOG__POLARIS__SCOPE="PRINCIPAL_ROLE:ALL" \
-	PYICEBERG_CATALOG__POLARIS__WAREHOUSE="polaris" \
-	PYICEBERG_CATALOG__POLARIS__HEADER__X_ICEBERG_ACCESS_DELEGATION="vended-credentials" \
-	PYICEBERG_CATALOG__POLARIS__HEADER__REALM="POLARIS" \
-	PYICEBERG_CATALOG__POLARIS__S3__ENDPOINT="http://localhost:9000" \
-	PYICEBERG_CATALOG__POLARIS__S3__ACCESS_KEY_ID="admin" \
-	PYICEBERG_CATALOG__POLARIS__S3__SECRET_ACCESS_KEY="password" \
-	PYICEBERG_CATALOG__POLARIS__S3__REGION="us-east-1" \
-	$(TEST_RUNNER) pytest tests/integration/test_catalog.py -k "rest_test_catalog and not test_update_namespace_properties" $(PYTEST_ARGS)
-	# Skip test_update_namespace_properties: Polaris triggers a CommitConflictException when updates and removals are in the same request.
-	# TODO: Remove test exception after https://github.com/apache/polaris/pull/4013 is merged.
+	TEST_RUNNER="$(TEST_RUNNER)" sh dev/test-polaris.sh $(PYTEST_ARGS)
 
 test-polaris-cleanup: ## Clean up Polaris integration test environment
 	@if [ "${KEEP_COMPOSE}" != "1" ]; then \

dev/docker-compose-polaris.yml

@@ -16,61 +16,246 @@
 # under the License.
 
 services:
+
+  rustfs:
+    image: rustfs/rustfs:1.0.0-alpha.81
+    networks:
+      iceberg_net:
+    ports:
+      # API port
+      - "9000:9000"
+      # UI port
+      - "9001:9001"
+    environment:
+      RUSTFS_ACCESS_KEY: polaris_root
+      RUSTFS_SECRET_KEY: polaris_pass
+      RUSTFS_VOLUMES: /data
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_CONSOLE_ENABLE: "true"
+      RUSTFS_CONSOLE_ADDRESS: ":9001"
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health && curl --fail http://127.0.0.1:9001/rustfs/console/health"]
+      interval: 10s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
   polaris:
     image: apache/polaris:latest
     container_name: pyiceberg-polaris
     networks:
       iceberg_net:
     ports:
-      - 8181:8181
-      - 8182:8182
+      # API port
+      - "8181:8181"
+      # Management port (metrics and health checks)
+      - "8182:8182"
+      # Optional, allows attaching a debugger to the Polaris JVM
+      - "5005:5005"
+    depends_on:
+      rustfs:
+        condition: service_healthy
+      bucket-setup:
+        condition: service_completed_successfully
     environment:
-      - POLARIS_BOOTSTRAP_CREDENTIALS=POLARIS,root,s3cr3t
-      - polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true
-      - polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3"]
-      - polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true
-      - polaris.readiness.ignore-severe-issues=true
-      - AWS_ACCESS_KEY_ID=admin
-      - AWS_SECRET_ACCESS_KEY=password
-      - AWS_REGION=us-east-1
+      JAVA_DEBUG: true
+      JAVA_DEBUG_PORT: "*:5005"
+      AWS_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: polaris_root
+      AWS_SECRET_ACCESS_KEY: polaris_pass
+      POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
+      polaris.realm-context.realms: POLARIS
+      quarkus.otel.sdk.disabled: "true"
+      polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true"
+      polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": '["FILE","S3"]'
+      polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS": "true"
+      polaris.readiness.ignore-severe-issues: "true"
     healthcheck:
-      test: ["CMD", "curl", "http://localhost:8182/q/health"]
-      interval: 10s
+      test: ["CMD", "curl", "--fail", "http://localhost:8182/q/health"]
+      interval: 2s
       timeout: 10s
-      retries: 5
-  minio:
-    image: minio/minio
-    container_name: pyiceberg-polaris-minio
+      retries: 10
+      start_period: 10s
+
+  bucket-setup:
+    image: amazon/aws-cli:2.34.19
     networks:
       iceberg_net:
-        aliases:
-          - warehouse.minio
-    ports:
-      - 9001:9001
-      - 9000:9000
+    depends_on:
+      rustfs:
+        condition: service_healthy
     environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-      - MINIO_DOMAIN=minio
-    command: ["server", "/data", "--console-address", ":9001"]
-  mc:
-    image: minio/mc
-    container_name: pyiceberg-polaris-mc
+      AWS_ACCESS_KEY_ID: polaris_root
+      AWS_SECRET_ACCESS_KEY: polaris_pass
+      AWS_ENDPOINT_URL: http://rustfs:9000
+    entrypoint: "/bin/sh"
+    command:
+      - "-c"
+      - >-
+        echo Creating RustFS bucket... &&
+        aws s3 mb s3://bucket123 &&
+        aws s3 ls &&
+        echo Bucket setup complete.
+
+  polaris-setup:
+    image: alpine/curl:8.17.0
     networks:
       iceberg_net:
     depends_on:
-      - minio
+      polaris:
+        condition: service_healthy
     environment:
-      - AWS_ACCESS_KEY_ID=admin
-      - AWS_SECRET_ACCESS_KEY=password
-      - AWS_REGION=us-east-1
-    entrypoint: >
-      /bin/sh -c "
-      until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
-      /usr/bin/mc mb minio/warehouse;
-      /usr/bin/mc policy set public minio/warehouse;
-      tail -f /dev/null
-      "
+      - CLIENT_ID=root
+      - CLIENT_SECRET=s3cr3t
+      - CATALOG_NAME=polaris
+      - REALM=POLARIS
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        set -e
+        apk add --no-cache jq
+
+        echo "Obtaining root access token..."
+        TOKEN_RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/catalog/v1/oauth/tokens \
+          -H 'Content-Type: application/x-www-form-urlencoded' \
+          -d "grant_type=client_credentials&client_id=$${CLIENT_ID}&client_secret=$${CLIENT_SECRET}&scope=PRINCIPAL_ROLE:ALL" 2>&1) || {
+          echo "❌ Failed to obtain access token"
+          echo "$$TOKEN_RESPONSE" >&2
+          exit 1
+        }
+
+        TOKEN=$$(echo $$TOKEN_RESPONSE | jq -r '.access_token')
+        if [ -z "$$TOKEN" ] || [ "$$TOKEN" = "null" ]; then
+          echo "❌ Failed to parse access token from response"
+          echo "$$TOKEN_RESPONSE"
+          exit 1
+        fi
+        echo "✅ Obtained access token"
+
+        echo "Creating catalog '$$CATALOG_NAME' in realm $$REALM..."
+        PAYLOAD='{
+          "catalog": {
+            "name": "'$$CATALOG_NAME'",
+            "type": "INTERNAL",
+            "readOnly": false,
+            "properties": {
+              "default-base-location": "s3://bucket123",
+              "polaris.config.drop-with-purge.enabled": "true"
+            },
+            "storageConfigInfo": {
+              "storageType": "S3",
+              "allowedLocations": ["s3://bucket123"],
+              "endpoint": "http://localhost:9000",
+              "endpointInternal": "http://rustfs:9000",
+              "pathStyleAccess": true
+            }
+          }
+        }'
+
+        RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Accept: application/json" \
+          -H "Content-Type: application/json" \
+          -H "Polaris-Realm: $$REALM" \
+          -d "$$PAYLOAD" 2>&1) && echo -n "" || {
+          echo "❌ Failed to create catalog"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Catalog created"
+
+        echo ""
+        echo "Creating principal 'quickstart_user'..."
+        PRINCIPAL_RESPONSE=$$(curl --fail-with-body -s -X POST http://polaris:8181/api/management/v1/principals \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"principal": {"name": "quickstart_user", "properties": {}}}' 2>&1) || {
+          echo "❌ Failed to create principal"
+          echo "$$PRINCIPAL_RESPONSE" >&2
+          exit 1
+        }
+
+        USER_CLIENT_ID=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientId')
+        USER_CLIENT_SECRET=$$(echo $$PRINCIPAL_RESPONSE | jq -r '.credentials.clientSecret')
+        if [ -z "$$USER_CLIENT_ID" ] || [ "$$USER_CLIENT_ID" = "null" ] || [ -z "$$USER_CLIENT_SECRET" ] || [ "$$USER_CLIENT_SECRET" = "null" ]; then
+          echo "❌ Failed to parse user credentials from response"
+          echo "$$PRINCIPAL_RESPONSE"
+          exit 1
+        fi
+        echo "✅ Principal created with clientId: $$USER_CLIENT_ID"
+
+        echo "Creating principal role 'quickstart_user_role'..."
+        RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/principal-roles \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"principalRole": {"name": "quickstart_user_role", "properties": {}}}' 2>&1) && echo -n "" || {
+          echo "❌ Failed to create principal role"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Principal role created"
+
+        echo "Creating catalog role 'quickstart_catalog_role'..."
+        RESPONSE=$$(curl --fail-with-body -s -S -X POST http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"catalogRole": {"name": "quickstart_catalog_role", "properties": {}}}' 2>&1) && echo -n "" || {
+          echo "❌ Failed to create catalog role"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Catalog role created"
+
+        echo "Assigning principal role to principal..."
+        RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principals/quickstart_user/principal-roles \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"principalRole": {"name": "quickstart_user_role"}}' 2>&1) && echo -n "" || {
+          echo "❌ Failed to assign principal role"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Principal role assigned"
+
+        echo "Assigning catalog role to principal role..."
+        RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/principal-roles/quickstart_user_role/catalog-roles/$$CATALOG_NAME \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"catalogRole": {"name": "quickstart_catalog_role"}}' 2>&1) && echo -n "" || {
+          echo "❌ Failed to assign catalog role"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Catalog role assigned"
+
+        echo "Granting CATALOG_MANAGE_CONTENT privilege..."
+        RESPONSE=$$(curl --fail-with-body -s -S -X PUT http://polaris:8181/api/management/v1/catalogs/$$CATALOG_NAME/catalog-roles/quickstart_catalog_role/grants \
+          -H "Authorization: Bearer $$TOKEN" \
+          -H "Polaris-Realm: $$REALM" \
+          -H "Content-Type: application/json" \
+          -d '{"type": "catalog", "privilege": "CATALOG_MANAGE_CONTENT"}' 2>&1) && echo -n "" || {
+          echo "❌ Failed to grant privileges"
+          echo "$$RESPONSE" >&2
+          exit 1
+        }
+        echo "✅ Privileges granted"
+
+        echo "CLIENT_ID=$$USER_CLIENT_ID" > /tmp/polaris_creds.env
+        echo "CLIENT_SECRET=$$USER_CLIENT_SECRET" >> /tmp/polaris_creds.env
+
+        touch /tmp/polaris-setup-done
+        tail -f /dev/null
+    healthcheck:
+      interval: 1s
+      timeout: 10s
+      retries: 240
+      test: ["CMD", "test", "-f", "/tmp/polaris-setup-done"]
 
 networks:
   iceberg_net: