Skip to content

Commit 60195fe

Browse files
harshachharshach
authored
Merge branch 'main' into fix/26775-quickstart-oidc-custom-params
2 parents 053b209 + c137ea9 commit 60195fe

114 files changed

Lines changed: 2637 additions & 6790 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/openmetadata-service-unit-tests.yml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,6 @@ jobs:
6666
needs: changes
6767
if: ${{ needs.changes.outputs.java == 'true' }}
6868
steps:
69-
- name: Require safe-to-test label
70-
if: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'safe to test') }}
71-
run: |
72-
echo "The 'safe to test' label is required before running OpenMetadata service unit tests."
73-
exit 1
74-
7569
- name: Checkout
7670
uses: actions/checkout@v4
7771
with:

‎bootstrap/sql/migrations/native/1.13.0/mysql/postDataMigrationSQLScript.sql‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,68 @@ UPDATE ingestion_pipeline_entity
22
SET json = JSON_REMOVE(json, '$.sourceConfig.config.computeMetrics')
33
WHERE JSON_EXTRACT(json, '$.sourceConfig.config.computeMetrics') IS NOT NULL
44
AND pipelineType = 'profiler';
5+
6+
-- Hard-delete ingestion pipelines for Iceberg services (must run before service migration)
7+
DELETE ipe FROM ingestion_pipeline_entity ipe
8+
JOIN dbservice_entity dse
9+
ON JSON_UNQUOTE(JSON_EXTRACT(ipe.json, '$.service.id')) = dse.id
10+
WHERE dse.serviceType = 'Iceberg'
11+
AND JSON_UNQUOTE(JSON_EXTRACT(ipe.json, '$.service.type')) = 'databaseService';
12+
13+
-- Migrate Iceberg database services to CustomDatabase (connector removed)
14+
-- serviceType is a GENERATED column derived from json, so only update json
15+
UPDATE dbservice_entity
16+
SET json = JSON_SET(
17+
json,
18+
'$.serviceType', 'CustomDatabase',
19+
'$.connection.config.type', 'CustomDatabase'
20+
)
21+
WHERE serviceType = 'Iceberg';
22+
23+
-- Migrate serviceType in child entities (serviceType is in JSON blob only, no generated column)
24+
UPDATE database_entity
25+
SET json = JSON_SET(json, '$.serviceType', 'CustomDatabase')
26+
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Iceberg';
27+
28+
UPDATE database_schema_entity
29+
SET json = JSON_SET(json, '$.serviceType', 'CustomDatabase')
30+
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Iceberg';
31+
32+
UPDATE table_entity
33+
SET json = JSON_SET(json, '$.serviceType', 'CustomDatabase')
34+
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Iceberg';
35+
36+
UPDATE stored_procedure_entity
37+
SET json = JSON_SET(json, '$.serviceType', 'CustomDatabase')
38+
WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Iceberg';
39+
40+
41+
-- Migrate existing glossary term RELATED_TO relationships to include relationType
42+
-- For backward compatibility, existing relations without a relationType are set to "relatedTo"
43+
44+
UPDATE entity_relationship
45+
SET json = JSON_SET(COALESCE(json, '{}'), '$.relationType', 'relatedTo')
46+
WHERE fromEntity = 'glossaryTerm'
47+
AND toEntity = 'glossaryTerm'
48+
AND relation = 15
49+
AND (json IS NULL OR JSON_EXTRACT(json, '$.relationType') IS NULL);
50+
51+
-- Insert default glossary term relation settings if they don't exist
52+
-- This preserves any existing user customizations
53+
INSERT INTO openmetadata_settings (configType, json)
54+
SELECT 'glossaryTermRelationSettings', '{"relationTypes":[{"name":"relatedTo","displayName":"Related To","description":"General association between terms that are conceptually connected.","rdfPredicate":"https://open-metadata.org/ontology/relatedTo","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#1890ff"},{"name":"synonym","displayName":"Synonym","description":"Terms that have the same meaning and can be used interchangeably.","rdfPredicate":"http://www.w3.org/2004/02/skos/core#exactMatch","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"equivalence","isSystemDefined":true,"color":"#722ed1"},{"name":"antonym","displayName":"Antonym","description":"Terms that have opposite meanings.","rdfPredicate":"https://open-metadata.org/ontology/antonym","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#f5222d"},{"name":"broader","displayName":"Broader","description":"A more general term (hypernym).","inverseRelation":"narrower","rdfPredicate":"http://www.w3.org/2004/02/skos/core#broader","isSymmetric":false,"isTransitive":true,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#597ef7"},{"name":"narrower","displayName":"Narrower","description":"A more specific term (hyponym).","inverseRelation":"broader","rdfPredicate":"http://www.w3.org/2004/02/skos/core#narrower","isSymmetric":false,"isTransitive":true,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#85a5ff"},{"name":"partOf","displayName":"Part Of","description":"This term is a part or component of another term.","inverseRelation":"hasPart","rdfPredicate":"https://open-metadata.org/ontology/partOf","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#13c2c2"},{"name":"hasPart","displayName":"Has Part","description":"This term has the other term as a part or component.","inverseRelation":"partOf","rdfPredicate":"https://open-metadata.org/ontology/hasPart","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#36cfc9"},{"name":"calculatedFrom","displayName":"Calculated From","description":"This term/metric is calculated or derived from another term.","inverseRelation":"usedToCalculate","rdfPredicate":"https://open-metadata.org/ontology/calculatedFrom","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#faad14"},{"name":"usedToCalculate","displayName":"Used To Calculate","description":"This term is used in the calculation of another term.","inverseRelation":"calculatedFrom","rdfPredicate":"https://open-metadata.org/ontology/usedToCalculate","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#ffc53d"},{"name":"seeAlso","displayName":"See Also","description":"Related term that may provide additional context.","rdfPredicate":"http://www.w3.org/2000/01/rdf-schema#seeAlso","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#eb2f96"}]}'
55+
WHERE NOT EXISTS (
56+
SELECT 1 FROM openmetadata_settings WHERE configType = 'glossaryTermRelationSettings'
57+
);
58+
59+
-- Strip stale relatedTerms from glossary term entity JSON.
60+
-- relatedTerms is now loaded from entity_relationship table, not from entity JSON.
61+
-- Old data stored relatedTerms as EntityReference objects which fail to deserialize as TermRelation.
62+
UPDATE glossary_term_entity
63+
SET json = JSON_REMOVE(json, '$.relatedTerms')
64+
WHERE JSON_EXTRACT(json, '$.relatedTerms') IS NOT NULL;
65+
66+
-- Backfill conceptMappings for existing glossary terms
67+
UPDATE glossary_term_entity
68+
SET json = JSON_SET(COALESCE(json, '{}'), '$.conceptMappings', JSON_ARRAY())
69+
WHERE JSON_EXTRACT(json, '$.conceptMappings') IS NULL;

‎bootstrap/sql/migrations/native/1.13.0/postgres/postDataMigrationSQLScript.sql‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,68 @@ UPDATE ingestion_pipeline_entity
22
SET json = (json::jsonb #- '{sourceConfig,config,computeMetrics}')::json
33
WHERE json::jsonb -> 'sourceConfig' -> 'config' -> 'computeMetrics' IS NOT NULL
44
AND pipelineType = 'profiler';
5+
6+
-- Hard-delete ingestion pipelines for Iceberg services (must run before service migration)
7+
DELETE FROM ingestion_pipeline_entity ipe
8+
USING dbservice_entity dse
9+
WHERE dse.serviceType = 'Iceberg'
10+
AND ipe.json::jsonb -> 'service' ->> 'type' = 'databaseService'
11+
AND ipe.json::jsonb -> 'service' ->> 'id' = dse.id;
12+
13+
-- Migrate Iceberg database services to CustomDatabase (connector removed)
14+
-- serviceType is a GENERATED column derived from json, so only update json
15+
UPDATE dbservice_entity
16+
SET json = jsonb_set(
17+
jsonb_set(
18+
json::jsonb,
19+
'{serviceType}', '"CustomDatabase"'
20+
),
21+
'{connection,config,type}', '"CustomDatabase"'
22+
)::json
23+
WHERE serviceType = 'Iceberg';
24+
25+
-- Migrate serviceType in child entities (serviceType is in JSON blob only, no generated column)
26+
UPDATE database_entity
27+
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
28+
WHERE json->>'serviceType' = 'Iceberg';
29+
30+
UPDATE database_schema_entity
31+
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
32+
WHERE json->>'serviceType' = 'Iceberg';
33+
34+
UPDATE table_entity
35+
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
36+
WHERE json->>'serviceType' = 'Iceberg';
37+
38+
UPDATE stored_procedure_entity
39+
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
40+
WHERE json->>'serviceType' = 'Iceberg';
41+
-- Migrate existing glossary term RELATED_TO relationships to include relationType
42+
-- For backward compatibility, existing relations without a relationType are set to "relatedTo"
43+
44+
UPDATE entity_relationship
45+
SET json = jsonb_set(COALESCE(json::jsonb, '{}'::jsonb), '{relationType}', '"relatedTo"')
46+
WHERE fromentity = 'glossaryTerm'
47+
AND toentity = 'glossaryTerm'
48+
AND relation = 15
49+
AND (json IS NULL OR json::jsonb->>'relationType' IS NULL);
50+
51+
-- Insert default glossary term relation settings if they don't exist
52+
-- This preserves any existing user customizations
53+
INSERT INTO openmetadata_settings (configtype, json)
54+
SELECT 'glossaryTermRelationSettings', '{"relationTypes":[{"name":"relatedTo","displayName":"Related To","description":"General association between terms that are conceptually connected.","rdfPredicate":"https://open-metadata.org/ontology/relatedTo","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#1890ff"},{"name":"synonym","displayName":"Synonym","description":"Terms that have the same meaning and can be used interchangeably.","rdfPredicate":"http://www.w3.org/2004/02/skos/core#exactMatch","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"equivalence","isSystemDefined":true,"color":"#722ed1"},{"name":"antonym","displayName":"Antonym","description":"Terms that have opposite meanings.","rdfPredicate":"https://open-metadata.org/ontology/antonym","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#f5222d"},{"name":"broader","displayName":"Broader","description":"A more general term (hypernym).","inverseRelation":"narrower","rdfPredicate":"http://www.w3.org/2004/02/skos/core#broader","isSymmetric":false,"isTransitive":true,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#597ef7"},{"name":"narrower","displayName":"Narrower","description":"A more specific term (hyponym).","inverseRelation":"broader","rdfPredicate":"http://www.w3.org/2004/02/skos/core#narrower","isSymmetric":false,"isTransitive":true,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#85a5ff"},{"name":"partOf","displayName":"Part Of","description":"This term is a part or component of another term.","inverseRelation":"hasPart","rdfPredicate":"https://open-metadata.org/ontology/partOf","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#13c2c2"},{"name":"hasPart","displayName":"Has Part","description":"This term has the other term as a part or component.","inverseRelation":"partOf","rdfPredicate":"https://open-metadata.org/ontology/hasPart","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"hierarchical","isSystemDefined":true,"color":"#36cfc9"},{"name":"calculatedFrom","displayName":"Calculated From","description":"This term/metric is calculated or derived from another term.","inverseRelation":"usedToCalculate","rdfPredicate":"https://open-metadata.org/ontology/calculatedFrom","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#faad14"},{"name":"usedToCalculate","displayName":"Used To Calculate","description":"This term is used in the calculation of another term.","inverseRelation":"calculatedFrom","rdfPredicate":"https://open-metadata.org/ontology/usedToCalculate","isSymmetric":false,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#ffc53d"},{"name":"seeAlso","displayName":"See Also","description":"Related term that may provide additional context.","rdfPredicate":"http://www.w3.org/2000/01/rdf-schema#seeAlso","isSymmetric":true,"isTransitive":false,"isCrossGlossaryAllowed":true,"category":"associative","isSystemDefined":true,"color":"#eb2f96"}]}'::jsonb
55+
WHERE NOT EXISTS (
56+
SELECT 1 FROM openmetadata_settings WHERE configtype = 'glossaryTermRelationSettings'
57+
);
58+
59+
-- Strip stale relatedTerms from glossary term entity JSON.
60+
-- relatedTerms is now loaded from entity_relationship table, not from entity JSON.
61+
-- Old data stored relatedTerms as EntityReference objects which fail to deserialize as TermRelation.
62+
UPDATE glossary_term_entity
63+
SET json = (json::jsonb - 'relatedTerms')::json
64+
WHERE jsonb_exists(json::jsonb, 'relatedTerms');
65+
66+
-- Backfill conceptMappings for existing glossary terms
67+
UPDATE glossary_term_entity
68+
SET json = jsonb_set(COALESCE(json::jsonb, '{}'::jsonb), '{conceptMappings}', '[]'::jsonb)
69+
WHERE json IS NULL OR json::jsonb->'conceptMappings' IS NULL;

‎ingestion/airflow-constraints-2.10.5.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,6 @@ pydruid==0.6.9
574574
pyenchant==3.2.2
575575
pyexasol==0.27.0
576576
pygraphviz==1.14
577-
pyiceberg==0.8.1
578577
pykerberos==1.2.4
579578
pymongo==4.11
580579
pymssql==2.3.2

‎ingestion/setup.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666
"cassandra": "cassandra-driver>=3.28.0",
6767
"opensearch": "opensearch-py~=2.4.0",
6868
"starrocks": "pymysql~=1.0",
69-
"pyiceberg": "pyiceberg==0.5.1",
7069
"google-cloud-bigtable": "google-cloud-bigtable>=2.0.0",
7170
"google-cloud-pubsub": "google-cloud-pubsub>=2.0.0",
7271
"pyathena": "pyathena~=3.25.0",
@@ -296,14 +295,6 @@
296295
"thrift-sasl~=0.4",
297296
"impyla~=0.18.0",
298297
},
299-
"iceberg": {
300-
VERSIONS["pyiceberg"],
301-
# Forcing the version of a few packages so it plays nicely with other requirements.
302-
VERSIONS["pydantic"],
303-
VERSIONS["adlfs"],
304-
VERSIONS["gcsfs"],
305-
VERSIONS["pyarrow"],
306-
},
307298
"impala": {
308299
"presto-types-parser>=0.0.2",
309300
"impyla[kerberos]~=0.18.0",
@@ -463,7 +454,6 @@
463454
VERSIONS["cockroach"],
464455
# pydoris-custom pre-installed with --no-deps in Dockerfiles (SA<2 metadata constraint).
465456
VERSIONS["starrocks"],
466-
VERSIONS["pyiceberg"],
467457
"testcontainers==3.7.1;python_version<'3.9'",
468458
"testcontainers~=4.8.0;python_version>='3.9'",
469459
"minio==7.2.5",

‎ingestion/src/metadata/ingestion/models/patch_request.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ class PatchedEntity(BaseModel):
145145
"numberOfObjects": True,
146146
"size": True,
147147
"fileFormats": True,
148+
"extension": True,
148149
}
149150

150151
RESTRICT_UPDATE_LIST = [
@@ -153,6 +154,7 @@ class PatchedEntity(BaseModel):
153154
"owners",
154155
"displayName",
155156
"tableConstraints",
157+
"extension",
156158
]
157159

158160
ARRAY_ENTITY_FIELDS = ["columns", "tasks", "fields"]

‎ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,8 @@ def _create_datamodel_request(
264264
),
265265
sql=self._get_datamodel_sql_query(data_model=data_model),
266266
owners=self.get_owner_ref(dashboard_details=dashboard_details),
267-
project=self.get_project_name(dashboard_details=dashboard_details),
267+
project=data_model.projectName
268+
or self.get_project_name(dashboard_details=dashboard_details),
268269
)
269270
yield Either(right=data_model_request)
270271
self.register_record_datamodel(datamodel_request=data_model_request)
@@ -1144,12 +1145,20 @@ def get_column_info(self, data_source: DataSource) -> Optional[List[Column]]:
11441145
datasource_columns = []
11451146
for field in data_source.fields or []:
11461147
try:
1148+
description = field.description or ""
1149+
if field.formula:
1150+
formula_text = f"**Formula:** `{field.formula}`"
1151+
description = (
1152+
f"{description}\n\n{formula_text}"
1153+
if description
1154+
else formula_text
1155+
)
11471156
parsed_fields = {
11481157
"dataTypeDisplay": "Tableau Field",
11491158
"dataType": DataType.RECORD,
11501159
"name": truncate_column_name(field.id),
11511160
"displayName": field.name if field.name else field.id,
1152-
"description": field.description,
1161+
"description": description or None,
11531162
}
11541163
child_columns = self.get_child_columns(field=field)
11551164
if child_columns:

‎ingestion/src/metadata/ingestion/source/dashboard/tableau/models.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ class DatasourceField(BaseModel):
123123
name: Optional[str] = None
124124
upstreamColumns: Optional[List[Union[UpstreamColumn, None]]] = None
125125
description: Optional[str] = None
126+
formula: Optional[str] = None
126127

127128

128129
class UpstreamTableColumn(BaseModel):
@@ -158,6 +159,7 @@ class DataSource(BaseModel):
158159
id: str
159160
name: Optional[str] = None
160161
description: Optional[str] = None
162+
projectName: Optional[str] = None
161163
tags: Optional[List[TableauDataModelTag]] = []
162164
fields: Optional[List[DatasourceField]] = None
163165
upstreamTables: Optional[List[UpstreamTable]] = None

‎ingestion/src/metadata/ingestion/source/dashboard/tableau/queries.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
id
2828
name
2929
description
30+
... on PublishedDatasource {{
31+
projectName
32+
}}
3033
tags {{
3134
name
3235
}}
@@ -39,6 +42,9 @@
3942
remoteType
4043
}}
4144
description
45+
... on CalculatedField {{
46+
formula
47+
}}
4248
}}
4349
}}
4450
fields {{
@@ -50,6 +56,9 @@
5056
remoteType
5157
}}
5258
description
59+
... on CalculatedField {{
60+
formula
61+
}}
5362
}}
5463
upstreamTables {{
5564
id

0 commit comments

Comments
 (0)