Skip to content

Commit 4cee4f6

Browse files
FernandezAstoraarthy-dk
FernandezAstor
authored and
aarthy-dk
committed
feat(TG-1085): add data_classification tag field
Adds a data_classification tag field to the data model, supporting three-level inheritance (column → table → table_group) throughout the application, consistent with existing tag fields. - Migration 0193: ADD COLUMN data_classification to table_groups, data_table_chars, data_column_chars, score_definition_results_breakdown; recreate 8 scoring views with COALESCE inheritance - Schema setup: ADD COLUMN and COALESCE in all 8 standard views - ORM: data_classification added to TableGroup model and ScoreDefinitionBreakdownItem / ScoreCategory / SCORE_CATEGORIES - Queries: COALESCE inheritance in profiling anomaly, test result, and scoring queries; added to TAG_FIELDS and scoring categories list - Data Catalog: fix tag inheritance in list view (join table_groups in get_table_group_columns); fix CSV export to COALESCE from table_groups; add seeded filter defaults (Public/Internal/Confidential/Restricted) via merge_tag_defaults; Excel export column - Table Group Form: Data Classification input in TaggingForm - JS: TAG_KEYS/TAG_HELP in metadata_tags.js; three-level fallback chain in data_catalog.js - Reports: data_classification in PDF hygiene and test result report metadata - Import: "data classification" header mapped in import_metadata_dialog - Tests: 5 unit tests for merge_tag_defaults dedupe logic Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> test: update TAG_FIELDS count to 9 after adding data_classification Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> fix(TG-1085): address round-2 review comments
1 parent 8e3d36e commit 4cee4f6

33 files changed

Lines changed: 194 additions & 21 deletions

‎testgen/common/data_catalog_service.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"transform_level",
3737
"aggregation_level",
3838
"data_product",
39+
"data_classification",
3940
]
4041

4142
# Metadata fields settable per target type, keyed by their data_*_chars column names.

‎testgen/common/models/data_column.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,7 @@ class DataColumnChars(Entity):
270270
transform_level: str | None = Column(String(40))
271271
aggregation_level: str | None = Column(String(40))
272272
data_product: str | None = Column(String(40))
273+
data_classification: str | None = Column(String(40))
273274
drop_date: datetime | None = Column(postgresql.TIMESTAMP)
274275
last_complete_profile_run_id: UUID | None = Column(postgresql.UUID(as_uuid=True))
275276
dq_score_profiling: float | None = Column(Float)

‎testgen/common/models/data_table.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ class DataTable(Entity):
7777
transform_level: str | None = Column(String(40))
7878
aggregation_level: str | None = Column(String(40))
7979
data_product: str | None = Column(String(40))
80+
data_classification: str | None = Column(String(40))
8081
drop_date: datetime | None = Column(postgresql.TIMESTAMP)
8182
last_complete_profile_run_id: UUID | None = Column(postgresql.UUID(as_uuid=True))
8283
dq_score_profiling: float | None = Column(Float)

‎testgen/common/models/scores.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"stakeholder_group",
5353
"transform_level",
5454
"data_product",
55+
"data_classification",
5556
]
5657
Categories = Literal[
5758
"column_name",
@@ -68,6 +69,7 @@
6869
"stakeholder_group",
6970
"transform_level",
7071
"data_product",
72+
"data_classification",
7173
]
7274
ScoreTypes = Literal["score", "cde_score"]
7375

@@ -88,6 +90,7 @@ class ScoreCategory(enum.Enum):
8890
dq_dimension = "dq_dimension"
8991
impact_dimension = "impact_dimension"
9092
data_product = "data_product"
93+
data_classification = "data_classification"
9194

9295

9396
class ScoreDefinition(Base):
@@ -826,6 +829,7 @@ class ScoreDefinitionBreakdownItem(Base):
826829
stakeholder_group: str = Column(String, nullable=True)
827830
transform_level: str = Column(String, nullable=True)
828831
data_product: str = Column(String, nullable=True)
832+
data_classification: str = Column(String, nullable=True)
829833
impact: float = Column(Float)
830834
score: float = Column(Float)
831835
issue_ct: int = Column(Integer)

‎testgen/common/models/table_group.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ class TableGroup(Entity):
261261
stakeholder_group: str = Column(NullIfEmptyString)
262262
transform_level: str = Column(NullIfEmptyString)
263263
data_product: str = Column(NullIfEmptyString)
264+
data_classification: str = Column(NullIfEmptyString)
264265
last_complete_profile_run_id: UUID = Column(postgresql.UUID(as_uuid=True))
265266
dq_score_profiling: float = Column(Float)
266267
dq_score_testing: float = Column(Float)

‎testgen/mcp/tools/common.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ class ScoreGroupBy(StrEnum):
158158
STAKEHOLDER_GROUP = "Stakeholder Group"
159159
TRANSFORM_LEVEL = "Transform Level"
160160
DATA_PRODUCT = "Data Product"
161+
DATA_CLASSIFICATION = "Data Classification"
161162

162163

163164
# Translates the user-facing label to the internal DB column name used by
@@ -175,6 +176,7 @@ class ScoreGroupBy(StrEnum):
175176
ScoreGroupBy.STAKEHOLDER_GROUP: "stakeholder_group",
176177
ScoreGroupBy.TRANSFORM_LEVEL: "transform_level",
177178
ScoreGroupBy.DATA_PRODUCT: "data_product",
179+
ScoreGroupBy.DATA_CLASSIFICATION: "data_classification",
178180
}
179181

180182

@@ -197,6 +199,7 @@ class ScoreFilterField(StrEnum):
197199
STAKEHOLDER_GROUP = "Stakeholder Group"
198200
TRANSFORM_LEVEL = "Transform Level"
199201
DATA_PRODUCT = "Data Product"
202+
DATA_CLASSIFICATION = "Data Classification"
200203

201204

202205
SCORE_FILTER_FIELD_TO_COLUMN: dict[ScoreFilterField, str] = {
@@ -210,6 +213,7 @@ class ScoreFilterField(StrEnum):
210213
ScoreFilterField.STAKEHOLDER_GROUP: "stakeholder_group",
211214
ScoreFilterField.TRANSFORM_LEVEL: "transform_level",
212215
ScoreFilterField.DATA_PRODUCT: "data_product",
216+
ScoreFilterField.DATA_CLASSIFICATION: "data_classification",
213217
}
214218

215219

@@ -233,6 +237,7 @@ class ScoreCategoryArg(StrEnum):
233237
QUALITY_DIMENSION = "Quality Dimension"
234238
IMPACT_DIMENSION = "Impact Dimension"
235239
DATA_PRODUCT = "Data Product"
240+
DATA_CLASSIFICATION = "Data Classification"
236241

237242

238243
SCORE_CATEGORY_ARG_TO_COLUMN: dict[ScoreCategoryArg, str] = {
@@ -247,6 +252,7 @@ class ScoreCategoryArg(StrEnum):
247252
ScoreCategoryArg.QUALITY_DIMENSION: "dq_dimension",
248253
ScoreCategoryArg.IMPACT_DIMENSION: "impact_dimension",
249254
ScoreCategoryArg.DATA_PRODUCT: "data_product",
255+
ScoreCategoryArg.DATA_CLASSIFICATION: "data_classification",
250256
}
251257

252258

‎testgen/mcp/tools/data_catalog.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ def update_catalog_metadata(updates: list[dict]) -> str:
4848
pii: Whether the column contains personally identifiable information
4949
(true/false). Columns only.
5050
data_source, source_system, source_process, business_domain,
51-
stakeholder_group, transform_level, aggregation_level, data_product:
51+
stakeholder_group, transform_level, aggregation_level, data_product,
52+
data_classification:
5253
Catalog tags (max 40 characters each).
5354
"""
5455
if not updates:

‎testgen/mcp/tools/quality_scores.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ def get_quality_scores(
8787
``"Table Group"``, ``"Data Location"``, ``"Data Source"``,
8888
``"Source System"``, ``"Source Process"``, ``"Business Domain"``,
8989
``"Stakeholder Group"``, ``"Transform Level"``, ``"Semantic Data Type"``,
90-
``"Data Product"``. To target specific tables or columns, chain a
90+
``"Data Product"``, ``"Data Classification"``. To target specific tables or columns, chain a
9191
``"Table Group"`` filter via ``others`` into ``"Table"`` (optionally
9292
then ``"Column"``); sibling chains OR. ``"Impact Dimension"`` and
9393
``"Quality Dimension"`` are valid as ``group_by`` only, not as filter
@@ -106,7 +106,7 @@ def get_quality_scores(
106106
``"Table Group"``, ``"Data Location"``, ``"Data Source"``,
107107
``"Source System"``, ``"Source Process"``, ``"Business Domain"``,
108108
``"Stakeholder Group"``, ``"Transform Level"``,
109-
``"Data Product"``.
109+
``"Data Product"``, ``"Data Classification"``.
110110
score_type: Narrow returned scores. Omit to show all four (Total,
111111
CDE, Profiling, Testing); pass ``"Total"`` for Total + Profiling
112112
+ Testing, or ``"CDE"`` for CDE alone.
@@ -551,7 +551,7 @@ def create_scorecard(
551551
``"Table Group"``, ``"Data Location"``, ``"Data Source"``,
552552
``"Source System"``, ``"Source Process"``, ``"Business Domain"``,
553553
``"Stakeholder Group"``, ``"Transform Level"``, ``"Semantic Data Type"``,
554-
``"Data Product"``. To target specific tables or columns, chain a
554+
``"Data Product"``, ``"Data Classification"``. To target specific tables or columns, chain a
555555
``"Table Group"`` filter via ``others`` into ``"Table"`` (optionally
556556
then ``"Column"``); sibling chains OR.
557557
@@ -563,7 +563,8 @@ def create_scorecard(
563563
``"Quality Dimension"``, ``"Impact Dimension"``,
564564
``"Data Source"``, ``"Business Domain"``, ``"Stakeholder Group"``,
565565
``"Table Group"``, ``"Transform Level"``, ``"Data Location"``,
566-
``"Source System"``, ``"Source Process"``, ``"Data Product"``.
566+
``"Source System"``, ``"Source Process"``, ``"Data Product"``,
567+
``"Data Classification"``.
567568
show_total_score: Whether the scorecard exposes the Total Score.
568569
show_cde_score: Whether the scorecard exposes the CDE Score.
569570
"""
@@ -624,7 +625,7 @@ def update_scorecard(
624625
``"Table Group"``, ``"Data Location"``, ``"Data Source"``,
625626
``"Source System"``, ``"Source Process"``, ``"Business Domain"``,
626627
``"Stakeholder Group"``, ``"Transform Level"``, ``"Semantic Data Type"``,
627-
``"Data Product"``. To target specific tables or columns, chain a
628+
``"Data Product"``, ``"Data Classification"``. To target specific tables or columns, chain a
628629
``"Table Group"`` filter via ``others`` into ``"Table"`` (optionally
629630
then ``"Column"``); sibling chains OR.
630631
@@ -638,7 +639,8 @@ def update_scorecard(
638639
``"Quality Dimension"``, ``"Impact Dimension"``,
639640
``"Data Source"``, ``"Business Domain"``, ``"Stakeholder Group"``,
640641
``"Table Group"``, ``"Transform Level"``, ``"Data Location"``,
641-
``"Source System"``, ``"Source Process"``, ``"Data Product"``.
642+
``"Source System"``, ``"Source Process"``, ``"Data Product"``,
643+
``"Data Classification"``.
642644
Pass ``""`` to clear an existing category.
643645
filters: List of filter entries. See **Filters** above for shape.
644646
"""

‎testgen/mcp/tools/table_groups.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ def create_table_group(
214214
stakeholder_group: str | None = None,
215215
transform_level: str | None = None,
216216
data_product: str | None = None,
217+
data_classification: str | None = None,
217218
) -> str:
218219
"""Create a table group on an existing connection.
219220
@@ -254,6 +255,8 @@ def create_table_group(
254255
``Conformed``, ``Processed``, ``Reporting``) or Medallion level
255256
(``bronze``, ``silver``, ``gold``).
256257
data_product: Catalog tag — data domain that comprises the dataset.
258+
data_classification: Catalog tag — information classification level of the dataset
259+
(e.g. ``Public``, ``Internal``, ``Confidential``, ``Restricted``).
257260
"""
258261
connection = resolve_connection(connection_id)
259262

@@ -288,6 +291,7 @@ def create_table_group(
288291
stakeholder_group=stakeholder_group,
289292
transform_level=transform_level,
290293
data_product=data_product,
294+
data_classification=data_classification,
291295
)
292296

293297
errors = validate_table_group_fields(table_group)
@@ -332,6 +336,7 @@ def update_table_group(
332336
stakeholder_group: str | None = None,
333337
transform_level: str | None = None,
334338
data_product: str | None = None,
339+
data_classification: str | None = None,
335340
) -> str:
336341
"""Update fields on an existing table group. Atomic — no partial save.
337342
@@ -371,6 +376,8 @@ def update_table_group(
371376
``Conformed``, ``Processed``, ``Reporting``) or Medallion level
372377
(``bronze``, ``silver``, ``gold``).
373378
data_product: Catalog tag — data domain that comprises the dataset.
379+
data_classification: Catalog tag — information classification level of the dataset
380+
(e.g. ``Public``, ``Internal``, ``Confidential``, ``Restricted``).
374381
"""
375382
supplied = {
376383
"table_group_name": table_group_name,
@@ -397,6 +404,7 @@ def update_table_group(
397404
"stakeholder_group": stakeholder_group,
398405
"transform_level": transform_level,
399406
"data_product": data_product,
407+
"data_classification": data_classification,
400408
}
401409
if all(value is None for value in supplied.values()):
402410
raise MCPUserError("No fields supplied to update.")
@@ -545,6 +553,7 @@ def preview_table_group(
545553
"stakeholder_group",
546554
"transform_level",
547555
"data_product",
556+
"data_classification",
548557
)
549558

550559
_DIFF_LABELS: dict[str, str] = {
@@ -572,6 +581,7 @@ def preview_table_group(
572581
"stakeholder_group": "Stakeholder group",
573582
"transform_level": "Transform level",
574583
"data_product": "Data product",
584+
"data_classification": "Data classification",
575585
}
576586

577587
_CATALOG_ATTRS: tuple[str, ...] = (
@@ -583,6 +593,7 @@ def preview_table_group(
583593
"stakeholder_group",
584594
"transform_level",
585595
"data_product",
596+
"data_classification",
586597
)
587598

588599

@@ -646,6 +657,7 @@ def _apply_args_to_table_group(
646657
stakeholder_group: str | None = None,
647658
transform_level: str | None = None,
648659
data_product: str | None = None,
660+
data_classification: str | None = None,
649661
) -> None:
650662
"""Apply every non-None arg to its model field.
651663
@@ -700,6 +712,8 @@ def _apply_args_to_table_group(
700712
table_group.transform_level = transform_level
701713
if data_product is not None:
702714
table_group.data_product = data_product
715+
if data_classification is not None:
716+
table_group.data_classification = data_classification
703717

704718

705719
def _raise_validation_error(errors: list[str], header: str) -> None:

‎testgen/template/dbsetup/030_initialize_new_schema_structure.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ CREATE TABLE table_groups
131131
stakeholder_group VARCHAR(40),
132132
transform_level VARCHAR(40),
133133
data_product VARCHAR(40),
134+
data_classification VARCHAR(40),
134135
last_complete_profile_run_id UUID,
135136
dq_score_profiling FLOAT,
136137
dq_score_testing FLOAT
@@ -429,6 +430,7 @@ CREATE TABLE data_table_chars (
429430
transform_level VARCHAR(40),
430431
aggregation_level VARCHAR(40),
431432
data_product VARCHAR(40),
433+
data_classification VARCHAR(40),
432434
add_date TIMESTAMP,
433435
drop_date TIMESTAMP,
434436
last_refresh_date TIMESTAMP,
@@ -468,6 +470,7 @@ CREATE TABLE data_column_chars (
468470
transform_level VARCHAR(40),
469471
aggregation_level VARCHAR(40),
470472
data_product VARCHAR(40),
473+
data_classification VARCHAR(40),
471474
add_date TIMESTAMP,
472475
last_mod_date TIMESTAMP,
473476
drop_date TIMESTAMP,
@@ -851,6 +854,7 @@ CREATE TABLE IF NOT EXISTS score_definition_results_breakdown (
851854
stakeholder_group TEXT DEFAULT NULL,
852855
transform_level TEXT DEFAULT NULL,
853856
data_product TEXT DEFAULT NULL,
857+
data_classification TEXT DEFAULT NULL,
854858
impact DOUBLE PRECISION DEFAULT 0.0,
855859
score DOUBLE PRECISION DEFAULT 0.0,
856860
issue_ct INTEGER DEFAULT 0

0 commit comments

Comments
 (0)