Skip to content

Commit ac81679

Browse files
fix(ingestion): replace class-name allowlist with module-path check for hostPort stripping
- Replace _HOSTPORT_URL_ALLOWED frozenset (only 2 entries) with _DATABASE_CONNECTION_MODULE_MARKER that checks the class module path. Only classes under .services.connections.database. are subject to scheme stripping; dashboard, pipeline, metadata, and all other connector categories pass through unchanged because their hostPort legitimately holds a full URL (e.g. http://grafana:3000). - Add isinstance(host_port, str) guard in model_post_init to prevent AttributeError when hostPort is Optional and left unset (None). - Update test_connection_builders.py: replace _HOSTPORT_URL_ALLOWED import with _DATABASE_CONNECTION_MODULE_MARKER, rename and update the allowlist test to reflect the new module-path strategy, and add a regression test for the None hostPort crash (CassandraConnection with default hostPort). Fixes gitar-bot Bug 1 (None crash) and Bug 2 (incomplete class-name allowlist that failed to cover non-database URL-based connectors).
1 parent d813bd6 commit ac81679

3 files changed

Lines changed: 49 additions & 24 deletions

File tree

ingestion/src/metadata/ingestion/connections/builders.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"""
1313
Get and test connection utilities
1414
"""
15+
1516
from functools import partial
1617
from typing import Any, Callable, Dict, Optional
1718
from urllib.parse import quote_plus

ingestion/src/metadata/ingestion/models/custom_pydantic.py

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
dependencies against any other metadata package. This class should
1616
be self-sufficient with only pydantic at import time.
1717
"""
18+
1819
import json
1920
import logging
2021
from typing import Any, Callable, Dict, Literal, Optional, Union
@@ -122,12 +123,12 @@ def strip_hostport_scheme(raw: str) -> str:
122123
# pinned to the original underscored symbol while the helper was private.
123124
_strip_hostport_scheme = strip_hostport_scheme
124125

125-
# Connection classes whose hostPort field legitimately stores a full URL
126-
# (e.g. "http://airflow:8080" or "http://localhost:8585/api") and must NOT
127-
# have their scheme stripped by model_post_init.
128-
_HOSTPORT_URL_ALLOWED: frozenset = frozenset(
129-
{"AirflowConnection", "OpenMetadataConnection"}
130-
)
126+
# Module sub-path that identifies database-connector classes.
127+
# Only *Connection classes whose module contains this sub-path have a plain
128+
# ``hostname[:port]`` hostPort — all other connection categories
129+
# (dashboard, pipeline, search, metadata, …) legitimately store a full URL
130+
# in hostPort and must NOT have their scheme stripped.
131+
_DATABASE_CONNECTION_MODULE_MARKER = ".services.connections.database."
131132

132133

133134
class BaseModel(PydanticBaseModel):
@@ -154,16 +155,21 @@ def model_post_init(self, context: Any, /):
154155
if not hasattr(self, "__pydantic_fields__"):
155156
return
156157

157-
# Strip accidental URL schemes from hostPort at model construction time.
158-
# This protects connectors that split hostPort manually (e.g. Cassandra,
159-
# Databricks, Redshift, DB2, Microsoft Fabric) without going through
160-
# get_connection_url_common. ValueError for genuinely invalid inputs
161-
# (e.g. non-numeric port) propagates immediately with a clear message.
158+
# Strip accidental URL schemes from hostPort at model construction time,
159+
# but ONLY for database-connector classes whose hostPort is a plain
160+
# ``hostname[:port]``. Dashboard, pipeline, search, and other connector
161+
# categories legitimately use a full URL in hostPort (e.g.
162+
# ``http://grafana:3000``), so we restrict stripping to classes whose
163+
# module path contains the database-connection marker.
164+
# The isinstance(str) guard prevents an AttributeError when hostPort is
165+
# optional and left unset (None).
166+
host_port = getattr(self, "hostPort", None)
162167
if (
163-
hasattr(self, "hostPort")
164-
and self.__class__.__name__ not in _HOSTPORT_URL_ALLOWED
168+
isinstance(host_port, str)
169+
and "://" in host_port
170+
and _DATABASE_CONNECTION_MODULE_MARKER in (self.__class__.__module__ or "")
165171
):
166-
self.hostPort = strip_hostport_scheme(self.hostPort)
172+
self.hostPort = strip_hostport_scheme(host_port)
167173

168174
try:
169175
for field in self.__pydantic_fields__:

ingestion/tests/unit/test_connection_builders.py

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,15 @@
1111
"""
1212
Validate connection builder utilities
1313
"""
14+
1415
from unittest import TestCase
1516

1617
from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
1718
BasicAuth,
1819
)
20+
from metadata.generated.schema.entity.services.connections.database.cassandraConnection import (
21+
CassandraConnection,
22+
)
1923
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
2024
MysqlConnection,
2125
)
@@ -27,7 +31,7 @@
2731
get_connection_options_dict,
2832
init_empty_connection_arguments,
2933
)
30-
from metadata.ingestion.models.custom_pydantic import _HOSTPORT_URL_ALLOWED
34+
from metadata.ingestion.models.custom_pydantic import _DATABASE_CONNECTION_MODULE_MARKER
3135

3236

3337
class ConnectionBuilderTest(TestCase):
@@ -127,20 +131,34 @@ def test_model_post_init_raises_for_invalid_port_in_hostport(self):
127131
hostPort="http://localhost:abc",
128132
)
129133

130-
def test_url_allowed_connections_not_stripped(self):
134+
def test_non_database_connections_not_stripped(self):
131135
"""
132-
Verify that connection classes in _HOSTPORT_URL_ALLOWED (AirflowConnection
133-
and OpenMetadataConnection) are excluded from hostPort scheme stripping,
134-
since their hostPort field legitimately stores a full URL.
136+
Verify that non-database connection classes (metadata, dashboard, pipeline, …)
137+
are NOT subject to hostPort scheme stripping because their hostPort legitimately
138+
stores a full URL (e.g. OpenMetadataConnection hostPort = http://localhost:8585/api).
139+
140+
The guard uses the module path: only classes whose __module__ contains
141+
_DATABASE_CONNECTION_MODULE_MARKER ('.services.connections.database.')
142+
are stripped. All others pass through unchanged.
135143
"""
136-
# Confirm the exclusion set contains the expected class names
137-
self.assertIn("AirflowConnection", _HOSTPORT_URL_ALLOWED)
138-
self.assertIn("OpenMetadataConnection", _HOSTPORT_URL_ALLOWED)
144+
# Confirm the marker is correct
145+
self.assertIn("database", _DATABASE_CONNECTION_MODULE_MARKER)
139146

140-
# OpenMetadataConnection.hostPort is a plain str field that expects a
141-
# full URL like "http://localhost:8585/api" — it must NOT be stripped.
147+
# OpenMetadataConnection.hostPort is a plain str that expects a full URL.
148+
# Its module contains '.connections.metadata.' — NOT the database marker —
149+
# so it must NOT be stripped.
142150
ometa = OpenMetadataConnection(
143151
hostPort="http://localhost:8585/api",
144152
)
145153
self.assertIn("http", ometa.hostPort)
146154
self.assertIn("localhost", ometa.hostPort)
155+
156+
def test_none_hostport_does_not_crash(self):
157+
"""
158+
Regression test for gitar-bot bug report: constructing a database
159+
connection where hostPort is Optional and left as None must NOT raise
160+
AttributeError from strip_hostport_scheme(None).
161+
"""
162+
# CassandraConnection.hostPort is Optional[str] = None by default.
163+
conn = CassandraConnection()
164+
self.assertIsNone(conn.hostPort)

0 commit comments

Comments
 (0)