Merge branch 'jgdoyon/disable-job-metadata-etl' into 'main'

fix(ontology): remove JobMetadata entity (source table not replicated)

See merge request gitlab-org/orbit/knowledge-graph!1750

Author: Jean-Gabriel Doyon

config/graph.sql

@@ -428,35 +428,6 @@ CREATE TABLE IF NOT EXISTS gl_job (
 ORDER BY (traversal_path, id) PRIMARY KEY (traversal_path, id)
 SETTINGS index_granularity = 1024, deduplicate_merge_projection_mode = 'rebuild', allow_experimental_replacing_merge_with_cleanup = 1, add_minmax_index_for_temporal_columns = 1, allow_part_offset_column_in_projections = 1, auto_statistics_types = 'minmax, uniq, countmin';
 
-CREATE TABLE IF NOT EXISTS gl_job_metadata (
-    id Int64 CODEC(Delta(8), LZ4),
-    build_id Int64 CODEC(Delta(8), LZ4),
-    interruptible Nullable(Bool),
-    timeout Nullable(Int64) CODEC(Delta(8), LZ4),
-    timeout_source LowCardinality(String) DEFAULT '' CODEC(LZ4),
-    exit_code Nullable(Int64) CODEC(Delta(8), LZ4),
-    has_exposed_artifacts Nullable(Bool),
-    expanded_environment_name String DEFAULT '' CODEC(ZSTD(1)),
-    environment_auto_stop_in String DEFAULT '' CODEC(ZSTD(1)),
-    debug_trace_enabled Bool DEFAULT false,
-    project_id Int64 CODEC(Delta(8), LZ4),
-    traversal_path String DEFAULT '0/' CODEC(ZSTD(1)),
-    _version DateTime64(6, 'UTC') DEFAULT now64(6) CODEC(ZSTD(1)),
-    _deleted Bool DEFAULT false,
-    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_build_id build_id TYPE bloom_filter(0.001) GRANULARITY 1,
-    INDEX idx_project_id project_id TYPE bloom_filter(0.001) GRANULARITY 1,
-    PROJECTION by_id (SELECT _part_offset ORDER BY (id, traversal_path, build_id, _version)),
-    PROJECTION by_build_id (SELECT _part_offset ORDER BY (build_id, traversal_path, _version)),
-    PROJECTION by_project_id (SELECT _part_offset ORDER BY (project_id, traversal_path, build_id, _version)),
-    PROJECTION tp_count (
-      SELECT traversal_path, uniq(id)
-      GROUP BY traversal_path
-    )
-) ENGINE = ReplacingMergeTree(_version, _deleted)
-ORDER BY (traversal_path, build_id) PRIMARY KEY (traversal_path, build_id)
-SETTINGS index_granularity = 1024, deduplicate_merge_projection_mode = 'rebuild', allow_experimental_replacing_merge_with_cleanup = 1, allow_part_offset_column_in_projections = 1, auto_statistics_types = 'minmax, uniq, countmin';
-
 CREATE TABLE IF NOT EXISTS gl_label (
     id Int64 CODEC(Delta(8), LZ4),
     title String DEFAULT '' CODEC(ZSTD(1)),

config/ontology/edges/has_metadata.yaml

@@ -161,11 +161,6 @@ settings:
     - {node: Job, property: type}
     - {node: Job, property: timeout_source}
     - {node: Job, property: scheduling_type}
-    # --- JobMetadata ---
-    - {node: JobMetadata, property: interruptible}
-    - {node: JobMetadata, property: timeout_source}
-    - {node: JobMetadata, property: has_exposed_artifacts}
-    - {node: JobMetadata, property: debug_trace_enabled}
     # --- MergeRequest ---
     - {node: MergeRequest, property: state}
     - {node: MergeRequest, property: merge_status}
@@ -349,7 +344,6 @@ domains:
       Pipeline: nodes/ci/pipeline.yaml
       Stage: nodes/ci/stage.yaml
       Job: nodes/ci/job.yaml
-      JobMetadata: nodes/ci/job_metadata.yaml
       Deployment: nodes/ci/deployment.yaml
       Environment: nodes/ci/environment.yaml
       Runner: nodes/ci/runner.yaml
@@ -442,5 +436,4 @@ edges:
   RUNS_FOR_PROJECT: edges/runs_for_project.yaml
   CHILD_OF: edges/child_of.yaml
   TRIGGERS_PIPELINE: edges/triggers_pipeline.yaml
-  HAS_METADATA: edges/has_metadata.yaml
   BUILT_BY: edges/built_by.yaml

config/ontology/nodes/ci/job_metadata.yaml

@@ -51,7 +51,6 @@ The Namespace Graph represents the software development lifecycle (SDLC) entitie
 | `MergeRequestDiffFile`| Represents a file inside a merge request diff.                                                          | `id`, `merge_request_id`, `merge_request_diff_id`, `new_path`, `old_path`, `generated`, `a_mode`, `b_mode` |
 | `Stage`               | Represents a CI stage.                                                                                  | `id`, `name`, `status`, `position`                                            |
 | `Job`                 | Represents a CI job (`Ci::Build` or `Ci::Bridge`).                                                      | `id`, `name`, `status`, `ref`, `allow_failure`, `type`, `runner_id`, `timeout`, `timeout_source`, `exit_code`, `scheduling_type`, `auto_canceled_by_id` |
-| `JobMetadata`         | Per-job runtime metadata sourced from `siphon_p_ci_builds_metadata`.                                    | `id`, `build_id`, `interruptible`, `timeout`, `timeout_source`, `exit_code`, `expanded_environment_name` |
 | `Runner`              | Represents a CI/CD runner (`Ci::Runner`). Global node, no `traversal_path` on the node table.            | `id`, `runner_type`, `name`, `active`, `locked`, `access_level`               |
 | `Finding`             | Represents a `security_findings` scan finding.                                                          | `id`, `uuid`, `name`, `severity`                                              |
 | `SecurityScan`        | Represents a security scan run.                                                                         | `id`, `scan_type`, `status`, `latest`                                         |
@@ -119,7 +118,6 @@ graph TD
 | `DISMISSED_BY`                      | `User`         | `Vulnerability`| A user dismissed a vulnerability.                                                                       |
 | `RESOLVED_BY`                       | `User`         | `Vulnerability`| A user resolved a vulnerability.                                                                        |
 | `HAS_JOB`                           | `Stage`, `Pipeline` | `Job`     | A stage contains jobs (canonical Pipeline → Stage → Job traversal); also exposed directly as Pipeline → Job for the natural CI mental model. |
-| `HAS_METADATA`                      | `Job`          | `JobMetadata`  | Job has runtime metadata (interruptible, effective timeout, expanded environment) sourced from `siphon_p_ci_builds_metadata`. |
 | `IN_PIPELINE`                       | `Job`, `SecurityScan` | `Pipeline` | A job or security scan belongs to a pipeline (one-hop replacement for `Pipeline → Stage → Job`).   |
 | `HAS_STAGE`                         | `Pipeline`     | `Stage`        | A pipeline contains stages.                                                                             |
 | `AUTO_CANCELED_BY`                  | `Pipeline`, `Job` | `Pipeline`, `Job` | Entity was auto-canceled when a newer entity of the same kind superseded it.                       |

config/ontology/schema.yaml

@@ -127,7 +127,6 @@ Examples from the current ontology:
 |---|---|---|
 | MergeRequest | `[traversal_path, id]` | `id` |
 | User (global) | `[id]` | `id` |
-| JobMetadata | `[traversal_path, build_id]` | `build_id` |
 
 Entities where the first non-scope column has low cardinality (e.g., Note's
 `noteable_type` with ~10 enum values) are poor partitioning candidates and

crates/integration-tests/tests/indexer/scenarios/sdlc/ci/processes_job_metadata.yaml

@@ -2,7 +2,7 @@
 stage: Analytics
 group: Knowledge Graph
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments
-description: Full reference for all 28 Orbit node types across 6 domains, including properties and their types.
+description: Full reference for all 27 Orbit node types across 6 domains, including properties and their types.
 title: Schema reference
 ---
 
@@ -26,7 +26,7 @@ title: Schema reference
 > For more information, see the history.
 > This feature is available for testing, but not ready for production use.
 
-Orbit indexes 28 node types across 6 domains. Use these as entity names in your queries.
+Orbit indexes 27 node types across 6 domains. Use these as entity names in your queries.
 
 To fetch the live schema at any time:
 
@@ -70,7 +70,6 @@ glab orbit remote schema
 | `Job` | CI/CD job | `id`, `name`, `status`, `ref`, `allow_failure`, `environment`, `failure_reason` |
 | `Deployment` | CI/CD deployment of a commit | `id`, `iid`, `status`, `ref`, `sha`, `environment_id` |
 | `Environment` | CI/CD deployment target | `id`, `name`, `state`, `tier`, `external_url` |
-| `JobMetadata` | Per-job runtime metadata | `id`, `build_id`, `interruptible`, `timeout`, `exit_code` |
 | `Runner` | CI/CD runner | `id`, `runner_type`, `name`, `active`, `locked` |
 
 ## Planning