From 694e350ef045f53d9ec3056c1adb2175fa121fce Mon Sep 17 00:00:00 2001 From: Abhipal Singh Date: Sun, 23 Mar 2025 12:09:28 +0000 Subject: [PATCH 1/2] Fixed setup of internal dbt project used by Elementary --- elementary/monitor/cli.py | 30 ++++++++++++++++++++++++++++-- elementary/monitor/dbt_init.py | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 elementary/monitor/dbt_init.py diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index 019293f1f..feab7a22b 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -11,6 +11,7 @@ ) from elementary.monitor.data_monitoring.schema import FiltersSchema from elementary.monitor.data_monitoring.selector_filter import SelectorFilter +from elementary.monitor.dbt_init import DBTInit from elementary.monitor.debug import Debug from elementary.tracking.anonymous_tracking import AnonymousCommandLineTracking from elementary.utils import bucket_path @@ -24,6 +25,7 @@ class Command: REPORT = "monitor-report" SEND_REPORT = "monitor-send-report" DEBUG = "debug" + DBT_INIT = "dbt-init" # Displayed in reverse order in --help. @@ -378,7 +380,8 @@ def monitor( if not success: sys.exit(1) except Exception as exc: - anonymous_tracking.track_cli_exception(Command.MONITOR, exc, ctx.command.name) + anonymous_tracking.track_cli_exception( + Command.MONITOR, exc, ctx.command.name) raise @@ -475,7 +478,8 @@ def report( if not generated_report_successfully: sys.exit(1) except Exception as exc: - anonymous_tracking.track_cli_exception(Command.REPORT, exc, ctx.command.name) + anonymous_tracking.track_cli_exception( + Command.REPORT, exc, ctx.command.name) raise @@ -766,5 +770,27 @@ def debug(ctx, profiles_dir): anonymous_tracking.track_cli_end(Command.DEBUG, None, ctx.command.name) +@monitor.command() +@click.pass_context +def dbt_init(ctx): + """ + Initializes the Elementary internal dbt project by installing its dbt deps. + Run this command after installing EDR as part of builds or CI/CD pipelines when the target + environment does not have write permissions on disk or does not have internet connection. + This command is not needed in most cases as the dbt deps are installed automatically when running `edr monitor`. + """ + config = Config() + anonymous_tracking = AnonymousCommandLineTracking(config) + anonymous_tracking.track_cli_start(Command.DEBUG, None, ctx.command.name) + dbtinit = DBTInit() + success = dbtinit.setup_internal_dbt_packages() + if not success: + sys.exit(1) + click.echo( + "Elementary internal dbt project has been initialized successfully. " + ) + anonymous_tracking.track_cli_end(Command.DEBUG, None, ctx.command.name) + + if __name__ == "__main__": monitor() diff --git a/elementary/monitor/dbt_init.py b/elementary/monitor/dbt_init.py new file mode 100644 index 000000000..333cb82c1 --- /dev/null +++ b/elementary/monitor/dbt_init.py @@ -0,0 +1,22 @@ +from elementary.clients.dbt.factory import create_dbt_runner +from elementary.config.config import Config +from elementary.monitor import dbt_project_utils + + +class DBTInit: + """ + Class to handle the initialization of dbt for the Elementary CLI. + This can contain all dbt static setup to avoid pulling in runtime dependencies or any other information from internet + that is internally used by edr. + """ + + def setup_internal_dbt_packages(self): + """ + Run dbt deps to install internal dbt packages if needed. + It intentionally does not use self.config.run_dbt_deps_if_needed parameter in create_dbt_runner to ensure + that dbt deps is always run when setting up the internal dbt packages. + """ + dbt_runner = create_dbt_runner( + dbt_project_utils.CLI_DBT_PROJECT_PATH + ) + return dbt_runner.deps() From b326ed863572c22e5c23fcf8ee4a6318bc2f6f74 Mon Sep 17 00:00:00 2001 From: Abhipal Singh Date: Mon, 7 Apr 2025 01:31:39 +0000 Subject: [PATCH 2/2] Precommit fixes --- elementary/monitor/cli.py | 12 ++++-------- elementary/monitor/dbt_init.py | 5 +---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/elementary/monitor/cli.py b/elementary/monitor/cli.py index feab7a22b..8412adb58 100644 --- a/elementary/monitor/cli.py +++ b/elementary/monitor/cli.py @@ -380,8 +380,7 @@ def monitor( if not success: sys.exit(1) except Exception as exc: - anonymous_tracking.track_cli_exception( - Command.MONITOR, exc, ctx.command.name) + anonymous_tracking.track_cli_exception(Command.MONITOR, exc, ctx.command.name) raise @@ -478,8 +477,7 @@ def report( if not generated_report_successfully: sys.exit(1) except Exception as exc: - anonymous_tracking.track_cli_exception( - Command.REPORT, exc, ctx.command.name) + anonymous_tracking.track_cli_exception(Command.REPORT, exc, ctx.command.name) raise @@ -774,7 +772,7 @@ def debug(ctx, profiles_dir): @click.pass_context def dbt_init(ctx): """ - Initializes the Elementary internal dbt project by installing its dbt deps. + Initializes the Elementary internal dbt project by installing its dbt deps. Run this command after installing EDR as part of builds or CI/CD pipelines when the target environment does not have write permissions on disk or does not have internet connection. This command is not needed in most cases as the dbt deps are installed automatically when running `edr monitor`. @@ -786,9 +784,7 @@ def dbt_init(ctx): success = dbtinit.setup_internal_dbt_packages() if not success: sys.exit(1) - click.echo( - "Elementary internal dbt project has been initialized successfully. " - ) + click.echo("Elementary internal dbt project has been initialized successfully. ") anonymous_tracking.track_cli_end(Command.DEBUG, None, ctx.command.name) diff --git a/elementary/monitor/dbt_init.py b/elementary/monitor/dbt_init.py index 333cb82c1..d1615de86 100644 --- a/elementary/monitor/dbt_init.py +++ b/elementary/monitor/dbt_init.py @@ -1,5 +1,4 @@ from elementary.clients.dbt.factory import create_dbt_runner -from elementary.config.config import Config from elementary.monitor import dbt_project_utils @@ -16,7 +15,5 @@ def setup_internal_dbt_packages(self): It intentionally does not use self.config.run_dbt_deps_if_needed parameter in create_dbt_runner to ensure that dbt deps is always run when setting up the internal dbt packages. """ - dbt_runner = create_dbt_runner( - dbt_project_utils.CLI_DBT_PROJECT_PATH - ) + dbt_runner = create_dbt_runner(dbt_project_utils.CLI_DBT_PROJECT_PATH) return dbt_runner.deps()