[DOP-31903] Bound Airflow task dependencies to the same parent DAG

Author: dolfinus

data_rentgen/consumer/extractors/impl/airflow_task.py

@@ -227,6 +227,7 @@ def _enrich_job_dependencies(self, run: RunDTO, event: OpenLineageRunEvent) -> R
                             name=f"{dag_info.dag_id}.{upstream_task_id}",
                             location=run.job.location,
                             type=JobTypeDTO("AIRFLOW_TASK"),
+                            parent_job=run.job.parent_job,
                         ),
                         to_job=run.job,
                         type="DIRECT_DEPENDENCY",
@@ -240,6 +241,7 @@ def _enrich_job_dependencies(self, run: RunDTO, event: OpenLineageRunEvent) -> R
                             name=f"{dag_info.dag_id}.{downstream_task_id}",
                             location=run.job.location,
                             type=JobTypeDTO("AIRFLOW_TASK"),
+                            parent_job=run.job.parent_job,
                         ),
                         type="DIRECT_DEPENDENCY",
                     ),

data_rentgen/db/scripts/seed/spark_yarn.py

@@ -215,6 +215,11 @@ def generate_spark_run_yarn(
                 name="mart_layer_dag.mart_layer_task_dbt",
                 location=parent_run.job.location,
                 type=JobTypeDTO(type="AIRFLOW_TASK"),
+                parent_job=JobDTO(
+                    name="mart_layer_dag",
+                    location=parent_run.job.location,
+                    type=JobTypeDTO(type="AIRFLOW_DAG"),
+                ),
             ),
             to_job=parent_run.job,
             type="DIRECT_DEPENDENCY",