Feat: Nessie support (#3700)

Author: erindru

Makefile

@@ -191,7 +191,7 @@ spark-test: engine-spark-up
 	pytest -n auto -x -m "spark or pyspark" --retries 3 --junitxml=test-results/junit-spark.xml
 
 trino-test: engine-trino-up
-	pytest -n auto -x -m "trino or trino_iceberg or trino_delta" --retries 3 --junitxml=test-results/junit-trino.xml
+	pytest -n auto -x -m "trino or trino_iceberg or trino_delta or trino_nessie" --retries 3 --junitxml=test-results/junit-trino.xml
 
 #################
 # Cloud Engines #

docs/integrations/engines/trino.md

@@ -47,7 +47,11 @@ iceberg.catalog.type=hive_metastore
 
 **Note**: The Trino Iceberg Connector must be configured with an `iceberg.catalog.type` that supports views. At the time of this writing, this is `hive_metastore`, `glue`, and `rest`.
 
-The `jdbc` and `nessie` catalogs do not support views and are thus incompatible with SQLMesh.
+The `jdbc` and `nessie` iceberg catalog types do not support views and are thus incompatible with SQLMesh.
+
+!!! info "Nessie"
+    Nessie is supported when used as an Iceberg REST Catalog (`iceberg.catalog.type=rest`).
+    For more information on how to configure the Trino Iceberg connector for this, see the [Nessie documentation](https://projectnessie.org/nessie-latest/trino/).
 
 #### Delta Lake Connector Configuration
 

tests/core/engine_adapter/integration/config.yaml

@@ -41,6 +41,17 @@ gateways:
       retries: 20
     state_connection:
       type: duckdb
+  inttest_trino_nessie:
+    connection:
+      type: trino
+      host: {{ env_var('DOCKER_HOSTNAME', 'localhost') }}
+      port: 8080
+      user: admin
+      catalog: datalake_nessie
+      http_scheme: http
+      retries: 20
+    state_connection:
+      type: duckdb
   inttest_spark:
     connection:
       type: spark

tests/core/engine_adapter/integration/docker/_common-hive.yaml

@@ -35,7 +35,8 @@ services:
       /usr/bin/mc mb --quiet myminio/trino/testing_iceberg;
       /usr/bin/mc mb --quiet myminio/trino/testing_delta;
       /usr/bin/mc mb --quiet myminio/spark/datalake;
-      /usr/bin/mc mb --quiet myminio/spark/testing
+      /usr/bin/mc mb --quiet myminio/spark/testing;
+      /usr/bin/mc mb --quiet myminio/nessie/warehouse;
       "
     depends_on:
       - minio

tests/core/engine_adapter/integration/docker/compose.trino.yaml

@@ -83,4 +83,28 @@ services:
       HIVE_METASTORE_WAREHOUSE_DIR: s3://trino/datalake_delta
       <<: *hive_metastore_environments
     depends_on:
-      - metastore
+      - metastore
+
+  nessie:
+    image: ghcr.io/projectnessie/nessie:0.102.2
+    restart: on-failure
+    ports:
+      - '19120:19120'
+    environment:
+      nessie.version.store.type: JDBC2
+      nessie.version.store.persist.jdbc.datasource: postgresql
+      quarkus.datasource.postgresql.jdbc.url: jdbc:postgresql://metastore:5432/nessie
+      quarkus.datasource.postgresql.username: hive
+      quarkus.datasource.postgresql.password: hive
+      nessie.catalog.default-warehouse: warehouse
+      nessie.catalog.warehouses.warehouse.location: s3://nessie/warehouse
+      nessie.catalog.service.s3.default-options.region: us-east-1
+      nessie.catalog.service.s3.default-options.path-style-access: 'true'
+      nessie.catalog.service.s3.default-options.access-key: urn:nessie-secret:quarkus:nessie.catalog.secrets.access-key
+      nessie.catalog.secrets.access-key.name: minio
+      nessie.catalog.secrets.access-key.secret: minio123
+      nessie.catalog.service.s3.default-options.endpoint: http://minio:9000/
+
+    depends_on:
+      - metastore
+

tests/core/engine_adapter/integration/docker/init-metastore-db.sql

@@ -10,4 +10,5 @@ create database datalake_iceberg_metastore;
 create database datalake_delta_metastore;
 create database testing_metastore;
 create database testing_iceberg_metastore;
-create database testing_delta_metastore;
+create database testing_delta_metastore;
+create database nessie;

tests/core/engine_adapter/integration/docker/trino/catalog/datalake_nessie.properties

@@ -0,0 +1,14 @@
+connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.security=NONE
+iceberg.rest-catalog.uri=http://nessie:19120/iceberg/
+iceberg.rest-catalog.vended-credentials-enabled=false
+iceberg.metadata-cache.enabled=false
+iceberg.hive-catalog-name=datalake
+
+fs.native-s3.enabled=true
+s3.endpoint=http://minio:9000
+s3.path-style-access=true
+s3.aws-access-key=minio
+s3.aws-secret-key=minio123
+s3.region=us-east-1

tests/core/engine_adapter/integration/test_integration.py

@@ -146,6 +146,14 @@ def test_type(request):
                 pytest.mark.trino_delta,
             ],
         ),
+        pytest.param(
+            "trino_nessie",
+            marks=[
+                pytest.mark.docker,
+                pytest.mark.engine,
+                pytest.mark.trino_nessie,
+            ],
+        ),
         pytest.param(
             "spark",
             marks=[