Skip to content

Commit 91e41b3

Browse files
feat: resolve adapter type from profiles.yml for transient error detection
- Add _get_adapter_type() method to CommandLineDbtRunner that parses dbt_project.yml and profiles.yml to resolve the actual adapter type (e.g. 'bigquery', 'snowflake') for the selected target. - Pass adapter_type instead of self.target to is_transient_error(), ensuring correct per-adapter pattern matching. - Remove duplicate 'databricks_catalog' entry from _ADAPTER_PATTERNS since profiles.yml always reports the adapter type, not the profile name. - Update docstrings to reflect that target should be the adapter type. - Gracefully falls back to None (check all patterns) if profiles cannot be parsed. Co-Authored-By: Itamar Hartstein <haritamar@gmail.com>
1 parent 3aeaf01 commit 91e41b3

2 files changed

Lines changed: 86 additions & 13 deletions

File tree

elementary/clients/dbt/command_line_dbt_runner.py

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,86 @@ def __init__(
8484
secret_vars,
8585
allow_macros_without_package_prefix,
8686
)
87+
self.adapter_type = self._get_adapter_type()
8788
self.raise_on_failure = raise_on_failure
8889
self.env_vars = env_vars
8990
if force_dbt_deps:
9091
self.deps()
9192
elif run_deps_if_needed:
9293
self._run_deps_if_needed()
9394

95+
def _get_adapter_type(self) -> Optional[str]:
96+
"""Resolve the adapter type from dbt profile configuration.
97+
98+
Reads ``dbt_project.yml`` to find the profile name, then looks up
99+
the selected target in ``profiles.yml`` to extract its ``type``
100+
field (e.g. ``"bigquery"``, ``"snowflake"``).
101+
102+
Returns ``None`` when the adapter type cannot be determined (missing
103+
files, missing keys, parse errors, etc.).
104+
"""
105+
try:
106+
# Read dbt_project.yml to get the profile name.
107+
dbt_project_path = os.path.join(self.project_dir, "dbt_project.yml")
108+
if not os.path.exists(dbt_project_path):
109+
logger.debug("dbt_project.yml not found at %s", dbt_project_path)
110+
return None
111+
112+
with open(dbt_project_path) as f:
113+
dbt_project = yaml.safe_load(f)
114+
115+
profile_name = dbt_project.get("profile")
116+
if not profile_name:
117+
logger.debug("No profile name found in dbt_project.yml")
118+
return None
119+
120+
# Determine profiles directory.
121+
if self.profiles_dir:
122+
profiles_dir = self.profiles_dir
123+
else:
124+
profiles_dir = os.path.join(os.path.expanduser("~"), ".dbt")
125+
126+
profiles_path = os.path.join(profiles_dir, "profiles.yml")
127+
if not os.path.exists(profiles_path):
128+
logger.debug("profiles.yml not found at %s", profiles_path)
129+
return None
130+
131+
with open(profiles_path) as f:
132+
profiles = yaml.safe_load(f)
133+
134+
profile = profiles.get(profile_name)
135+
if not profile:
136+
logger.debug("Profile '%s' not found in profiles.yml", profile_name)
137+
return None
138+
139+
# Determine which target to use.
140+
target_name = self.target if self.target else profile.get("target")
141+
if not target_name:
142+
logger.debug("No target specified and no default target in profile")
143+
return None
144+
145+
outputs = profile.get("outputs", {})
146+
target_config = outputs.get(target_name)
147+
if not target_config:
148+
logger.debug("Target '%s' not found in profile outputs", target_name)
149+
return None
150+
151+
adapter_type = target_config.get("type")
152+
if adapter_type:
153+
logger.debug(
154+
"Resolved adapter type '%s' for target '%s'",
155+
adapter_type,
156+
target_name,
157+
)
158+
else:
159+
logger.debug("No type found in target configuration")
160+
161+
return adapter_type
162+
163+
except Exception as exc:
164+
logger.debug("Failed to resolve adapter type: %s", exc)
165+
return None
166+
94167
def _inner_run_command(
95168
self,
96169
dbt_command_args: List[str],
@@ -203,7 +276,9 @@ def _inner_run_command_with_retries(
203276
if isinstance(exc.proc_err.stderr, bytes)
204277
else str(exc.proc_err.stderr)
205278
)
206-
if is_transient_error(self.target, output=output_text, stderr=stderr_text):
279+
if is_transient_error(
280+
self.adapter_type, output=output_text, stderr=stderr_text
281+
):
207282
raise DbtTransientError(
208283
result=DbtCommandResult(
209284
success=False,
@@ -225,7 +300,7 @@ def _inner_run_command_with_retries(
225300
logger.info(log.msg)
226301

227302
if not result.success and is_transient_error(
228-
self.target, output=result.output, stderr=result.stderr
303+
self.adapter_type, output=result.output, stderr=result.stderr
229304
):
230305
raise DbtTransientError(
231306
result=result,

elementary/clients/dbt/transient_errors.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@
1010
substrings that appear in the error output. Matching is
1111
case-insensitive substring search so regex is not needed.
1212
13-
Note: The ``target`` argument accepted by :func:`is_transient_error` may
14-
be either the dbt adapter type *or* the profile target name (e.g.
15-
``"dev"``, ``"prod"``). When it does not match any known adapter key,
16-
**all** adapter patterns are checked defensively. This is safe because
17-
adapter-specific error messages only appear in output from that adapter.
13+
The ``target`` argument accepted by :func:`is_transient_error` should be
14+
the dbt **adapter type** (e.g. ``"bigquery"``, ``"snowflake"``), as
15+
resolved from ``profiles.yml``. When the value does not match any known
16+
adapter key (or is ``None``), **all** adapter patterns are checked
17+
defensively so that transient errors are never missed.
1818
"""
1919

2020
from typing import Dict, Optional, Sequence, Tuple
@@ -76,7 +76,6 @@
7676
"ssl syscall error",
7777
),
7878
"databricks": _DATABRICKS_PATTERNS,
79-
"databricks_catalog": _DATABRICKS_PATTERNS,
8079
"athena": (
8180
"throttlingexception",
8281
"toomanyrequestsexception",
@@ -120,13 +119,12 @@ def is_transient_error(
120119
Parameters
121120
----------
122121
target:
123-
The dbt adapter type (e.g. ``"bigquery"``, ``"snowflake"``) **or**
124-
the dbt profile target name (e.g. ``"dev"``, ``"prod"``).
122+
The dbt adapter type (e.g. ``"bigquery"``, ``"snowflake"``),
123+
typically resolved from ``profiles.yml`` by the runner.
125124
When the value matches a key in ``_ADAPTER_PATTERNS``, only that
126125
adapter's patterns (plus ``_COMMON``) are used. When it does
127-
**not** match any known adapter, **all** adapter patterns are
128-
checked defensively to avoid missing transient errors.
129-
When ``None``, all adapter patterns are checked defensively.
126+
**not** match any known adapter (or is ``None``), **all** adapter
127+
patterns are checked defensively to avoid missing transient errors.
130128
output:
131129
The captured stdout of the dbt command (may be ``None``).
132130
stderr:

0 commit comments

Comments
 (0)