Add PR metadata audit fields to commit parquet

Author: apstenku123

cppmega_mlx/data/nanochat_pipeline/packed_rows_schema.py

@@ -17,12 +17,16 @@
     FILEPATH_STABLE_ID_COLUMN,
     FILE_LOCAL_COMMIT_INDEX_COLUMN,
     HAS_AMBIGUOUS_RECONSTRUCTION_COLUMN,
+    HAS_PR_DISCUSSION_COLUMN,
     HAS_RENAME_AMBIGUITY_COLUMN,
     HUNK_ID_PER_TOKEN_COLUMN,
     IS_MERGE_COMMIT_COLUMN,
     PARENT_COUNT_COLUMN,
     PARENT_HASHES_COLUMN,
     PLATFORM_IDS_COLUMN,
+    PR_DISCUSSION_CHARS_COLUMN,
+    PR_DISCUSSION_LINES_COLUMN,
+    PR_NUMBER_COLUMN,
     RAW_COMMIT_CHRONOLOGY_COLUMNS,
     REPO_COLUMN,
     REPO_STABLE_ID_COLUMN,
@@ -73,10 +77,18 @@
 SOURCE_REPO_STABLE_IDS_COLUMN = "source_repo_stable_ids"
 SOURCE_FILEPATH_STABLE_IDS_COLUMN = "source_filepath_stable_ids"
 SOURCE_FILE_LOCAL_COMMIT_INDICES_COLUMN = "source_file_local_commit_indices"
+SOURCE_PR_NUMBERS_COLUMN = "source_pr_numbers"
+SOURCE_HAS_PR_DISCUSSIONS_COLUMN = "source_has_pr_discussions"
+SOURCE_PR_DISCUSSION_CHARS_COLUMN = "source_pr_discussion_chars"
+SOURCE_PR_DISCUSSION_LINES_COLUMN = "source_pr_discussion_lines"
 PACKED_REPO_COLUMN = REPO_COLUMN
 PACKED_FILEPATH_COLUMN = FILEPATH_COLUMN
 PACKED_COMMIT_HASH_COLUMN = COMMIT_HASH_COLUMN
 PACKED_TIMESTAMP_COLUMN = TIMESTAMP_COLUMN
+PACKED_PR_NUMBER_COLUMN = PR_NUMBER_COLUMN
+PACKED_HAS_PR_DISCUSSION_COLUMN = HAS_PR_DISCUSSION_COLUMN
+PACKED_PR_DISCUSSION_CHARS_COLUMN = PR_DISCUSSION_CHARS_COLUMN
+PACKED_PR_DISCUSSION_LINES_COLUMN = PR_DISCUSSION_LINES_COLUMN
 PACKED_PARENT_HASHES_COLUMN = PARENT_HASHES_COLUMN
 PACKED_PARENT_COUNT_COLUMN = PARENT_COUNT_COLUMN
 PACKED_IS_MERGE_COMMIT_COLUMN = IS_MERGE_COMMIT_COLUMN
@@ -196,11 +208,19 @@
     SOURCE_REPO_STABLE_IDS_COLUMN,
     SOURCE_FILEPATH_STABLE_IDS_COLUMN,
     SOURCE_FILE_LOCAL_COMMIT_INDICES_COLUMN,
+    SOURCE_PR_NUMBERS_COLUMN,
+    SOURCE_HAS_PR_DISCUSSIONS_COLUMN,
+    SOURCE_PR_DISCUSSION_CHARS_COLUMN,
+    SOURCE_PR_DISCUSSION_LINES_COLUMN,
     ROW_PLATFORM_IDS_COLUMN,
     PACKED_REPO_COLUMN,
     PACKED_FILEPATH_COLUMN,
     PACKED_COMMIT_HASH_COLUMN,
     PACKED_TIMESTAMP_COLUMN,
+    PACKED_PR_NUMBER_COLUMN,
+    PACKED_HAS_PR_DISCUSSION_COLUMN,
+    PACKED_PR_DISCUSSION_CHARS_COLUMN,
+    PACKED_PR_DISCUSSION_LINES_COLUMN,
     PACKED_PARENT_HASHES_COLUMN,
     PACKED_PARENT_COUNT_COLUMN,
     PACKED_IS_MERGE_COMMIT_COLUMN,
@@ -347,12 +367,20 @@ def resolve_packed_row_valid_token_count(
     "SOURCE_REPO_STABLE_IDS_COLUMN",
     "SOURCE_FILEPATH_STABLE_IDS_COLUMN",
     "SOURCE_FILE_LOCAL_COMMIT_INDICES_COLUMN",
+    "SOURCE_PR_NUMBERS_COLUMN",
+    "SOURCE_HAS_PR_DISCUSSIONS_COLUMN",
+    "SOURCE_PR_DISCUSSION_CHARS_COLUMN",
+    "SOURCE_PR_DISCUSSION_LINES_COLUMN",
     "PLATFORM_IDS_COLUMN",
     "ROW_PLATFORM_IDS_COLUMN",
     "PACKED_REPO_COLUMN",
     "PACKED_FILEPATH_COLUMN",
     "PACKED_COMMIT_HASH_COLUMN",
     "PACKED_TIMESTAMP_COLUMN",
+    "PACKED_PR_NUMBER_COLUMN",
+    "PACKED_HAS_PR_DISCUSSION_COLUMN",
+    "PACKED_PR_DISCUSSION_CHARS_COLUMN",
+    "PACKED_PR_DISCUSSION_LINES_COLUMN",
     "PACKED_PARENT_HASHES_COLUMN",
     "PACKED_PARENT_COUNT_COLUMN",
     "PACKED_IS_MERGE_COMMIT_COLUMN",

cppmega_mlx/data/nanochat_pipeline/tokenized_enriched_schema.py

@@ -101,6 +101,9 @@ def _extract_span_bounds(span: object) -> tuple[int, int] | None:
 COMMIT_HASH_COLUMN = "commit_hash"
 TIMESTAMP_COLUMN = "timestamp"
 PR_NUMBER_COLUMN = "pr_number"
+HAS_PR_DISCUSSION_COLUMN = "has_pr_discussion"
+PR_DISCUSSION_CHARS_COLUMN = "pr_discussion_chars"
+PR_DISCUSSION_LINES_COLUMN = "pr_discussion_lines"
 PARENT_HASHES_COLUMN = "parent_hashes"
 PARENT_COUNT_COLUMN = "parent_count"
 IS_MERGE_COMMIT_COLUMN = "is_merge_commit"
@@ -353,6 +356,9 @@ def has_materialized_token_metadata(
     "COMMIT_HASH_COLUMN",
     "TIMESTAMP_COLUMN",
     "PR_NUMBER_COLUMN",
+    "HAS_PR_DISCUSSION_COLUMN",
+    "PR_DISCUSSION_CHARS_COLUMN",
+    "PR_DISCUSSION_LINES_COLUMN",
     "PARENT_HASHES_COLUMN",
     "PARENT_COUNT_COLUMN",
     "IS_MERGE_COMMIT_COLUMN",