diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63ee893fa2..9e61a8acaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,7 @@ jobs: env: # Disable debug info to speed up compilation and reduce artifact size RUSTFLAGS: "-C debuginfo=0" + ICEBERG_TEST_S3TABLES_ENDPOINT: "http://localhost:9000" run: | if [ "${{ matrix.test-suite.name }}" = "default" ]; then cargo nextest run ${{ matrix.test-suite.args }} diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index f19cf7bff4..2650cb4186 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -30,7 +30,7 @@ use iceberg_catalog_hms::{ HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{get_hms_endpoint, get_minio_endpoint, set_up}; +use iceberg_test_utils::{get_hms_endpoint, get_s3_endpoint, set_up}; use tokio::time::sleep; use tracing::info; @@ -40,7 +40,7 @@ async fn get_catalog() -> HmsCatalog { set_up(); let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -52,7 +52,7 @@ async fn get_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), s3_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs index 600cd9b6f4..c689f74011 100644 --- a/crates/catalog/loader/tests/common/mod.rs +++ b/crates/catalog/loader/tests/common/mod.rs @@ -49,7 +49,7 @@ use iceberg_catalog_sql::{ }; use iceberg_storage_opendal::OpenDalStorageFactory; use iceberg_test_utils::{ - get_glue_endpoint, get_hms_endpoint, get_minio_endpoint, get_rest_catalog_endpoint, set_up, + get_glue_endpoint, get_hms_endpoint, get_rest_catalog_endpoint, get_s3_endpoint, set_up, }; use sqlx::migrate::MigrateDatabase; use tempfile::TempDir; @@ -216,7 +216,7 @@ async fn rest_catalog() -> RestCatalog { async fn glue_catalog() -> GlueCatalog { let glue_endpoint = get_glue_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); let props = HashMap::from([ (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), @@ -225,7 +225,7 @@ async fn glue_catalog() -> GlueCatalog { "my_secret_key".to_string(), ), (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), s3_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), @@ -264,7 +264,7 @@ async fn glue_catalog() -> GlueCatalog { async fn hms_catalog() -> HmsCatalog { let hms_endpoint = get_hms_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); let props = HashMap::from([ (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), @@ -276,7 +276,7 @@ async fn hms_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), s3_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/catalog/s3tables/src/catalog.rs b/crates/catalog/s3tables/src/catalog.rs index b88bd77d29..0eb0f0b5c0 100644 --- a/crates/catalog/s3tables/src/catalog.rs +++ b/crates/catalog/s3tables/src/catalog.rs @@ -378,7 +378,7 @@ impl Catalog for S3TablesCatalog { match req.send().await { Ok(_) => Ok(true), Err(err) => { - if err.as_service_error().map(|e| e.is_not_found_exception()) == Some(true) { + if is_not_found_sdk_error(&err) { Ok(false) } else { Err(from_aws_sdk_error(err)) @@ -606,7 +606,7 @@ impl Catalog for S3TablesCatalog { match req.send().await { Ok(_) => Ok(true), Err(err) => { - if err.as_service_error().map(|e| e.is_not_found_exception()) == Some(true) { + if is_not_found_sdk_error(&err) { Ok(false) } else { Err(from_aws_sdk_error(err)) @@ -696,6 +696,35 @@ impl Catalog for S3TablesCatalog { } } +/// Check if an AWS SDK error represents a "not found" condition for a +/// namespace or table resource. +/// +/// The typed `NotFoundException` from the real AWS S3Tables API is matched by +/// checking the error code. S3Tables-compatible backends (e.g. SeaweedFS) may +/// return different error codes such as `NoSuchNamespace` or `NoSuchTable` +/// which the SDK deserializes as `Unhandled` variants. This function checks the +/// error code metadata to handle both cases. +/// +/// Importantly, this does NOT treat `NoSuchBucket` as "not found" -- a missing +/// table bucket is an infrastructure/configuration error that should propagate. +fn is_not_found_sdk_error(err: &aws_sdk_s3tables::error::SdkError) -> bool +where E: aws_sdk_s3tables::error::ProvideErrorMetadata { + match err { + aws_sdk_s3tables::error::SdkError::ServiceError(service_err) => { + matches!( + service_err.err().code(), + Some( + "NotFoundException" + | "NoSuchNamespace" + | "NoSuchTable" + | "ResourceNotFoundException" + ) + ) + } + _ => false, + } +} + /// Format AWS SDK error into iceberg error pub(crate) fn from_aws_sdk_error(error: aws_sdk_s3tables::error::SdkError) -> Error where T: std::fmt::Debug { @@ -712,6 +741,7 @@ mod tests { use super::*; + /// Load a catalog configured for real AWS S3Tables (requires TABLE_BUCKET_ARN). async fn load_s3tables_catalog_from_env() -> Result> { let table_bucket_arn = match std::env::var("TABLE_BUCKET_ARN").ok() { Some(table_bucket_arn) => table_bucket_arn, @@ -729,6 +759,41 @@ mod tests { Ok(Some(S3TablesCatalog::new(config, None).await?)) } + /// Load a catalog configured for local SeaweedFS S3Tables testing. + /// Falls back to real AWS if TABLE_BUCKET_ARN is set, otherwise checks + /// ICEBERG_TEST_S3TABLES_ENDPOINT for a local SeaweedFS instance. + async fn load_s3tables_catalog_for_test() -> Result> { + // Prefer real AWS if TABLE_BUCKET_ARN is set + if let Some(catalog) = load_s3tables_catalog_from_env().await? { + return Ok(Some(catalog)); + } + + // Fall back to local SeaweedFS via ICEBERG_TEST_S3TABLES_ENDPOINT + let endpoint_url = match std::env::var("ICEBERG_TEST_S3TABLES_ENDPOINT").ok() { + Some(url) => url, + None => return Ok(None), + }; + + let table_bucket_arn = std::env::var("TABLE_BUCKET_ARN").unwrap_or_else(|_| { + "arn:aws:s3tables:us-east-1:000000000000:bucket/iceberg-test".to_string() + }); + + let mut props = HashMap::new(); + props.insert("aws_access_key_id".to_string(), "admin".to_string()); + props.insert("aws_secret_access_key".to_string(), "password".to_string()); + props.insert("region_name".to_string(), "us-east-1".to_string()); + + let config = S3TablesCatalogConfig { + name: None, + table_bucket_arn, + endpoint_url: Some(endpoint_url), + client: None, + props, + }; + + Ok(Some(S3TablesCatalog::new(config, None).await?)) + } + #[tokio::test] async fn test_s3tables_list_namespace() { let catalog = match load_s3tables_catalog_from_env().await { @@ -776,7 +841,7 @@ mod tests { #[tokio::test] async fn test_s3tables_create_delete_namespace() { - let catalog = match load_s3tables_catalog_from_env().await { + let catalog = match load_s3tables_catalog_for_test().await { Ok(Some(catalog)) => catalog, Ok(None) => return, Err(e) => panic!("Error loading catalog: {e}"), @@ -794,7 +859,7 @@ mod tests { #[tokio::test] async fn test_s3tables_create_delete_table() { - let catalog = match load_s3tables_catalog_from_env().await { + let catalog = match load_s3tables_catalog_for_test().await { Ok(Some(catalog)) => catalog, Ok(None) => return, Err(e) => panic!("Error loading catalog: {e}"), @@ -821,8 +886,8 @@ mod tests { namespace.clone(), "test_s3tables_create_delete_table".to_string(), ); + catalog.purge_table(&table_ident).await.ok(); catalog.drop_namespace(&namespace).await.ok(); - catalog.drop_table(&table_ident).await.ok(); catalog .create_namespace(&namespace, HashMap::new()) @@ -830,14 +895,14 @@ mod tests { .unwrap(); catalog.create_table(&namespace, creation).await.unwrap(); assert!(catalog.table_exists(&table_ident).await.unwrap()); - catalog.drop_table(&table_ident).await.unwrap(); + catalog.purge_table(&table_ident).await.unwrap(); assert!(!catalog.table_exists(&table_ident).await.unwrap()); catalog.drop_namespace(&namespace).await.unwrap(); } #[tokio::test] async fn test_s3tables_update_table() { - let catalog = match load_s3tables_catalog_from_env().await { + let catalog = match load_s3tables_catalog_for_test().await { Ok(Some(catalog)) => catalog, Ok(None) => return, Err(e) => panic!("Error loading catalog: {e}"), @@ -849,7 +914,7 @@ mod tests { TableIdent::new(namespace.clone(), "test_s3tables_update_table".to_string()); // Clean up any existing resources from previous test runs - catalog.drop_table(&table_ident).await.ok(); + catalog.purge_table(&table_ident).await.ok(); catalog.drop_namespace(&namespace).await.ok(); // Create namespace and table diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index 4bf8f4d19c..6a1b8da5bf 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -20,7 +20,7 @@ use std::sync::OnceLock; use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; use iceberg_catalog_rest::REST_CATALOG_PROP_URI; -use iceberg_test_utils::{get_minio_endpoint, get_rest_catalog_endpoint, set_up}; +use iceberg_test_utils::{get_rest_catalog_endpoint, get_s3_endpoint, set_up}; /// Global test fixture that uses environment-based configuration. /// This assumes Docker containers are started externally (e.g., via `make docker-up`). @@ -37,11 +37,11 @@ impl GlobalTestFixture { set_up(); let rest_endpoint = get_rest_catalog_endpoint(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); let catalog_config = HashMap::from([ (REST_CATALOG_PROP_URI.to_string(), rest_endpoint), - (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ENDPOINT.to_string(), s3_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 207a4454d7..db92da7530 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -29,21 +29,21 @@ mod tests { FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_storage_opendal::{CustomAwsCredentialLoader, OpenDalStorageFactory}; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_s3_endpoint, normalize_test_name_with_parts, set_up}; use reqsign::{AwsCredential, AwsCredentialLoad}; use reqwest::Client; async fn get_file_io() -> FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { configured_scheme: "s3".to_string(), customized_credential_load: None, })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, s3_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -107,7 +107,7 @@ mod tests { Self { credential } } - fn new_minio() -> Self { + fn new_s3_test() -> Self { Self::new(Some(AwsCredential { access_key_id: "admin".to_string(), secret_access_key: "password".to_string(), @@ -127,7 +127,7 @@ mod tests { #[test] fn test_custom_aws_credential_loader_instantiation() { // Test creating CustomAwsCredentialLoader with mock loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_s3_test(); let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory @@ -147,10 +147,10 @@ mod tests { let _file_io = get_file_io().await; // Create a mock credential loader - let mock_loader = MockCredentialLoader::new_minio(); + let mock_loader = MockCredentialLoader::new_s3_test(); let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { @@ -158,7 +158,7 @@ mod tests { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, s3_endpoint), (S3_REGION, "us-east-1".to_string()), ]) .build(); @@ -178,7 +178,7 @@ mod tests { let mock_loader = MockCredentialLoader::new(None); let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); // Build FileIO with custom credential loader via OpenDalStorageFactory let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { @@ -186,7 +186,7 @@ mod tests { customized_credential_load: Some(custom_loader), })) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, s3_endpoint), (S3_REGION, "us-east-1".to_string()), ]) .build(); diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index 4572ad2c2d..b0d265415c 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -32,16 +32,16 @@ mod tests { FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; - use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{get_s3_endpoint, normalize_test_name_with_parts, set_up}; fn get_resolving_file_io() -> iceberg::io::FileIO { set_up(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, s3_endpoint), (S3_ACCESS_KEY_ID, "admin".to_string()), (S3_SECRET_ACCESS_KEY, "password".to_string()), (S3_REGION, "us-east-1".to_string()), @@ -260,10 +260,10 @@ mod tests { use reqsign::{AwsCredential, AwsCredentialLoad}; use reqwest::Client; - struct MinioCredentialLoader; + struct S3TestCredentialLoader; #[async_trait] - impl AwsCredentialLoad for MinioCredentialLoader { + impl AwsCredentialLoad for S3TestCredentialLoader { async fn load_credential( &self, _client: Client, @@ -278,15 +278,15 @@ mod tests { } set_up(); - let minio_endpoint = get_minio_endpoint(); + let s3_endpoint = get_s3_endpoint(); let factory = OpenDalResolvingStorageFactory::new().with_s3_credential_loader( - CustomAwsCredentialLoader::new(Arc::new(MinioCredentialLoader)), + CustomAwsCredentialLoader::new(Arc::new(S3TestCredentialLoader)), ); let file_io = FileIOBuilder::new(Arc::new(factory)) .with_props(vec![ - (S3_ENDPOINT, minio_endpoint), + (S3_ENDPOINT, s3_endpoint), (S3_REGION, "us-east-1".to_string()), ]) .build(); diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index e44d96c385..5a609c21aa 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -37,24 +37,33 @@ mod common { } // Environment variable names for service endpoints - pub const ENV_MINIO_ENDPOINT: &str = "ICEBERG_TEST_MINIO_ENDPOINT"; + pub const ENV_S3_ENDPOINT: &str = "ICEBERG_TEST_S3_ENDPOINT"; pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT"; pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; + pub const ENV_S3TABLES_ENDPOINT: &str = "ICEBERG_TEST_S3TABLES_ENDPOINT"; // Default ports matching dev/docker-compose.yaml - pub const DEFAULT_MINIO_PORT: u16 = 9000; + pub const DEFAULT_S3_PORT: u16 = 9000; pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181; pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; + pub const DEFAULT_S3TABLES_PORT: u16 = 9000; - /// Returns the MinIO S3-compatible endpoint. - /// Checks ICEBERG_TEST_MINIO_ENDPOINT env var, otherwise returns localhost default. - pub fn get_minio_endpoint() -> String { - std::env::var(ENV_MINIO_ENDPOINT) - .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_MINIO_PORT}")) + /// Returns the S3-compatible storage endpoint (SeaweedFS). + /// Checks ICEBERG_TEST_S3_ENDPOINT env var, otherwise returns localhost default. + pub fn get_s3_endpoint() -> String { + std::env::var(ENV_S3_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_S3_PORT}")) + } + + /// Returns the S3Tables endpoint (SeaweedFS). + /// S3Tables shares the same port as the S3 API on SeaweedFS. + pub fn get_s3tables_endpoint() -> String { + std::env::var(ENV_S3TABLES_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_S3TABLES_PORT}")) } /// Returns the REST catalog endpoint. diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 21920c9ce6..ea6fa1ade1 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,51 +25,59 @@ networks: services: # ============================================================================= - # MinIO - S3-compatible storage (shared by all tests) + # SeaweedFS - S3-compatible storage (shared by all tests) # ============================================================================= - minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - hostname: minio + seaweedfs: + image: chrislusf/seaweedfs:latest + hostname: seaweedfs networks: iceberg_test: - # Add aliases for virtual-hosted style bucket access aliases: - - icebergdata.minio - - warehouse.minio - - bucket1.minio + - icebergdata.seaweedfs + - warehouse.seaweedfs + - bucket1.seaweedfs ports: - "9000:9000" - - "9001:9001" - command: ["server", "/data", "--console-address", ":9001"] + - "9333:9333" + - "8888:8888" + volumes: + - ./seaweedfs/s3_iam.json:/etc/seaweedfs/s3_iam.json:ro + command: > + server -s3 -s3.port=9000 + -dir=/data + -volume.max=0 + -master.volumeSizeLimitMB=1024 + -master.port=9333 + -filer.port=8888 + -iam.config=/etc/seaweedfs/s3_iam.json + -ip=seaweedfs + -ip.bind=0.0.0.0 healthcheck: - test: ["CMD", "mc", "ready", "local"] + test: ["CMD", "curl", "-f", "http://localhost:9000/healthz"] interval: 5s timeout: 5s - retries: 5 + retries: 10 + start_period: 10s - # MinIO client - creates buckets for tests - mc: + # SeaweedFS init - creates buckets for tests + seaweedfs-init: depends_on: - minio: + seaweedfs: condition: service_healthy - image: minio/mc:RELEASE.2025-05-21T01-59-54Z + image: amazon/aws-cli:2.27.31 environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 + - AWS_DEFAULT_REGION=us-east-1 entrypoint: > /bin/sh -c " - /usr/bin/mc alias set minio http://minio:9000 admin password; - /usr/bin/mc mb --ignore-existing minio/icebergdata; - /usr/bin/mc mb --ignore-existing minio/warehouse; - /usr/bin/mc mb --ignore-existing minio/bucket1; - /usr/bin/mc policy set public minio/icebergdata; - /usr/bin/mc policy set public minio/warehouse; - /usr/bin/mc policy set public minio/bucket1; + aws --endpoint-url http://seaweedfs:9000 s3 mb s3://icebergdata || true; + aws --endpoint-url http://seaweedfs:9000 s3 mb s3://warehouse || true; + aws --endpoint-url http://seaweedfs:9000 s3 mb s3://bucket1 || true; + curl -sf -X POST http://seaweedfs:9000/ \ + -H 'Content-Type: application/x-amz-json-1.1' \ + -H 'X-Amz-Target: S3Tables.CreateTableBucket' \ + -d '{\"name\": \"iceberg-test\"}' || true; echo 'Buckets created successfully'; tail -f /dev/null " @@ -89,9 +97,9 @@ services: - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory - CATALOG_WAREHOUSE=s3://icebergdata/demo - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 + - CATALOG_S3_ENDPOINT=http://seaweedfs:9000 depends_on: - minio: + seaweedfs: condition: service_healthy networks: iceberg_test: @@ -113,7 +121,7 @@ services: dockerfile: Dockerfile platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} depends_on: - minio: + seaweedfs: condition: service_healthy networks: iceberg_test: @@ -170,7 +178,7 @@ services: depends_on: rest: condition: service_healthy - minio: + seaweedfs: condition: service_healthy environment: - AWS_ACCESS_KEY_ID=admin diff --git a/dev/hms/core-site.xml b/dev/hms/core-site.xml index f23efa8885..dbd6c7997b 100644 --- a/dev/hms/core-site.xml +++ b/dev/hms/core-site.xml @@ -30,7 +30,7 @@ fs.s3a.endpoint - http://minio:9000 + http://seaweedfs:9000 fs.s3a.access.key diff --git a/dev/seaweedfs/s3_iam.json b/dev/seaweedfs/s3_iam.json new file mode 100644 index 0000000000..624562e15e --- /dev/null +++ b/dev/seaweedfs/s3_iam.json @@ -0,0 +1,20 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "admin", + "secretKey": "password" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + } + ] +} diff --git a/dev/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf index b576f624b9..de1bb49eb8 100644 --- a/dev/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -23,7 +23,7 @@ spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://minio:9000 +spark.sql.catalog.rest.s3.endpoint http://seaweedfs:9000 spark.sql.catalog.rest.cache-enabled false spark.sql.defaultCatalog rest diff --git a/website/src/reference/container-runtimes.md b/website/src/reference/container-runtimes.md index 7b4ac38511..a354a979dc 100644 --- a/website/src/reference/container-runtimes.md +++ b/website/src/reference/container-runtimes.md @@ -19,7 +19,7 @@ # Container Runtimes -Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for MinIO and various catalogs. You can use any of the following container runtimes. +Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for SeaweedFS (S3-compatible storage with S3Tables support) and various catalogs. You can use any of the following container runtimes. ## Docker Desktop