Chore: Test retry flakiness and migrate pytest config to pyproject.toml (#4496)

Author: erindru

Makefile

@@ -170,7 +170,7 @@ clickhouse-cloud-test: guard-CLICKHOUSE_CLOUD_HOST guard-CLICKHOUSE_CLOUD_USERNA
 	pytest -n 1 -m "clickhouse_cloud" --retries 3 --junitxml=test-results/junit-clickhouse-cloud.xml
 
 athena-test: guard-AWS_ACCESS_KEY_ID guard-AWS_SECRET_ACCESS_KEY guard-ATHENA_S3_WAREHOUSE_LOCATION engine-athena-install
-	pytest -n auto -m "athena" --retries 3 --retry-delay 10 --junitxml=test-results/junit-athena.xml
+	pytest -n auto -m "athena" --retries 3 --junitxml=test-results/junit-athena.xml
 
 vscode_settings:
 	mkdir -p .vscode

pyproject.toml

@@ -215,6 +215,53 @@ module = [
 ]
 ignore_missing_imports = true
 
+[tool.pytest.ini_options]
+markers = [
+    # Test Type Markers
+    # Tests are ordered from fastest to slowest
+    "fast: fast tests (automatically applied if no type markers)",
+    "slow: slow tests that typically involve interacting with a local DB (like DuckDB)",
+    "docker: test that involves interacting with a Docker container",
+    "remote: test that involves interacting with a remote DB",
+    "cicdonly: test that only runs on CI/CD",
+    "isolated: tests that need to run sequentially usually because they use fork",
+
+    # Test Domain Markers
+    # default: core functionality
+    "cli: test for CLI",
+    "dbt: test for dbt adapter",
+    "github: test for Github CI/CD bot",
+    "jupyter: tests for Jupyter integration",
+    "web: tests for web UI",
+
+    # Engine Adapters
+    "engine: test all engine adapters",
+    "athena: test for Athena",
+    "bigquery: test for BigQuery",
+    "clickhouse: test for Clickhouse (standalone mode / cluster mode)",
+    "clickhouse_cloud: test for Clickhouse (cloud mode)",
+    "databricks: test for Databricks",
+    "duckdb: test for DuckDB",
+    "motherduck: test for MotherDuck",
+    "mssql: test for MSSQL",
+    "mysql: test for MySQL",
+    "postgres: test for Postgres",
+    "redshift: test for Redshift",
+    "snowflake: test for Snowflake",
+    "spark: test for Spark",
+    "trino: test for Trino (all connectors)",
+    "risingwave: test for Risingwave"
+]
+addopts = "-n 0 --dist=loadgroup"
+asyncio_default_fixture_loop_scope = "session"
+log_cli = false # Set this to true to enable logging during tests
+log_cli_format = "%(asctime)s.%(msecs)03d %(filename)s:%(lineno)d %(levelname)s %(message)s"
+log_cli_level = "INFO"
+filterwarnings = [
+    "ignore:The localize method is no longer necessary, as this time zone supports the fold attribute"
+]
+retry_delay = 10
+
 [tool.ruff.lint]
 select = [
     "F401",

pytest.ini

@@ -106,25 +106,23 @@ def _create(
             is_remote=is_remote,
         )
 
-        skip = False
         try:
             ctx.init()
-        except Exception:
-            # We need to catch this exception because if there is an error during setup, pytest-retry aborts immediately
-            # instead of retrying
+        except:
+            # pytest-retry doesnt work if there are errors in fixture setup (ref: https://github.com/str0zzapreti/pytest-retry/issues/33 )
+            # what we can do is log the exception and return a partially-initialized context to the test, which should
+            # throw an exception when it tries to access something that didnt init properly and thus trigger pytest-retry to retry
             logger.exception("Context init failed")
-            skip = True
-
-        if not skip:
-            with ctx.engine_adapter.session({}):
-                yield ctx
-
-            try:
-                ctx.cleanup()
-            except Exception:
-                # We need to catch this exception because if there is an error during teardown, pytest-retry aborts immediately
-                # instead of retrying
-                logger.exception("Context cleanup failed")
+
+        with ctx.engine_adapter.session({}):
+            yield ctx
+
+        try:
+            ctx.cleanup()
+        except:
+            # We need to catch this exception because if there is an error during teardown, pytest-retry aborts immediately
+            # instead of retrying
+            logger.exception("Context cleanup failed")
 
     return _create
 

tests/core/engine_adapter/integration/conftest.py

@@ -28,7 +28,7 @@ services:
 
   # Trino Stack
   trino:
-    image: 'trinodb/trino:465'
+    image: 'trinodb/trino:475'
     ports:
       - '8080:8080'
     volumes: