Skip to content

Commit 73468c1

Browse files
slevenickslevenick
authored
Dataplex metadata feed (#17979)
1 parent da4cd9c commit 73468c1

3 files changed

Lines changed: 320 additions & 0 deletions

File tree

‎mmv1/products/dataplex/MetadataFeed.yaml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# Copyright 2026 Google Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
---
15+
name: MetadataFeed
16+
description: A Dataplex Metadata Feed monitors Dataplex metadata entries in a specified scope and publishes notifications of changes to a Cloud Pub/Sub topic.
17+
base_url: projects/{{project}}/locations/{{location}}/metadataFeeds
18+
self_link: projects/{{project}}/locations/{{location}}/metadataFeeds/{{metadata_feed_id}}
19+
create_url: projects/{{project}}/locations/{{location}}/metadataFeeds?metadataFeedId={{metadata_feed_id}}
20+
update_mask: true
21+
update_verb: PATCH
22+
import_format:
23+
- projects/{{project}}/locations/{{location}}/metadataFeeds/{{metadata_feed_id}}
24+
async:
25+
operation:
26+
base_url: '{{op_id}}'
27+
actions:
28+
- create
29+
- update
30+
- delete
31+
result:
32+
resource_inside_response: true
33+
autogen_status: TWV0YWRhdGFGZWVkQXV0b2dlblYyQWdlbnQ=
34+
autogen_async: true
35+
samples:
36+
- name: dataplex_metadata_feed_basic
37+
primary_resource_id: primary
38+
steps:
39+
- name: dataplex_metadata_feed_basic
40+
resource_id_vars:
41+
metadata_feed_id: tf-test-feed
42+
vars:
43+
label_val: 'bar'
44+
test_env_vars:
45+
project_name: PROJECT_NAME
46+
- name: dataplex_metadata_feed_basic
47+
resource_id_vars:
48+
metadata_feed_id: tf-test-feed
49+
vars:
50+
label_val: 'baz'
51+
test_env_vars:
52+
project_name: PROJECT_NAME
53+
- name: dataplex_metadata_feed_projects
54+
primary_resource_id: primary
55+
exclude_basic_doc: true
56+
steps:
57+
- name: dataplex_metadata_feed_projects
58+
resource_id_vars:
59+
metadata_feed_id: tf-test-feed
60+
vars:
61+
label_val: 'bar'
62+
test_env_vars:
63+
project_name: PROJECT_NAME
64+
- name: dataplex_metadata_feed_projects
65+
resource_id_vars:
66+
metadata_feed_id: tf-test-feed
67+
vars:
68+
label_val: 'baz'
69+
test_env_vars:
70+
project_name: PROJECT_NAME
71+
parameters:
72+
- name: location
73+
type: String
74+
required: true
75+
description: Resource ID segment making up resource `name`. It identifies the resource within its parent collection as described in https://google.aip.dev/122.
76+
immutable: true
77+
url_param_only: true
78+
- name: metadataFeedId
79+
type: String
80+
required: true
81+
description: The metadata job ID. If not provided, a unique ID is generated with the prefix metadata-job-.
82+
immutable: true
83+
url_param_only: true
84+
properties:
85+
- name: createTime
86+
type: String
87+
description: The time when the feed was created.
88+
output: true
89+
- name: filters
90+
type: NestedObject
91+
description: Filters defines the type of changes that you want to listen to. You can have multiple entry type filters and multiple aspect type filters. All of the entry type filters are OR'ed together. All of the aspect type filters are OR'ed together. All of the entry type filters and aspect type filters are AND'ed together.
92+
properties:
93+
- name: aspectTypes
94+
type: Array
95+
description: 'The aspect types that you want to listen to. Depending on how the aspect is attached to the entry, in the format: projects/{project_id_or_number}/locations/{location}/aspectTypes/{aspect_type_id}.'
96+
item_type:
97+
type: String
98+
- name: changeTypes
99+
type: Array
100+
description: The type of change that you want to listen to. If not specified, all changes are published.
101+
item_type:
102+
type: String
103+
- name: entryTypes
104+
type: Array
105+
description: The entry types that you want to listen to, specified as relative resource names in the format projects/{project_id_or_number}/locations/{location}/entryTypes/{entry_type_id}. Only entries that belong to the specified entry types are published.
106+
item_type:
107+
type: String
108+
- name: labels
109+
type: KeyValueLabels
110+
description: User-defined labels.
111+
- name: name
112+
type: String
113+
description: Identifier. The resource name of the metadata feed, in the format projects/{project_id_or_number}/locations/{location_id}/metadataFeeds/{metadata_feed_id}.
114+
output: true
115+
- name: pubsubTopic
116+
type: String
117+
description: 'The pubsub topic that you want the metadata feed messages to publish to. Please grant Dataplex service account the permission to publish messages to the topic. The service account is: service-{PROJECT_NUMBER}@gcp-sa-dataplex.iam.gserviceaccount.com.'
118+
- name: scope
119+
type: NestedObject
120+
required: true
121+
description: Scope defines the scope of the metadata feed. Scopes are exclusive. Only one of the scopes can be specified.
122+
properties:
123+
- name: entryGroups
124+
type: Array
125+
description: 'The entry groups whose entries you want to listen to. Must be in the format: projects/{project_id_or_number}/locations/{location_id}/entryGroups/{entry_group_id}.'
126+
item_type:
127+
type: String
128+
- name: organizationLevel
129+
type: Boolean
130+
description: |-
131+
Whether the metadata feed is at the organization-level.
132+
If true, all changes happened to the entries in the same organization as the feed are published.
133+
If false, you must specify a list of projects or a list of entry groups whose entries you want to listen to.The default is false.
134+
- name: projects
135+
type: Array
136+
description: 'The projects whose entries you want to listen to. Must be in the same organization as the feed. Must be in the format: projects/{project_id_or_number}.'
137+
item_type:
138+
type: String
139+
- name: uid
140+
type: String
141+
description: A system-generated, globally unique ID for the metadata job. If the metadata job is deleted and then re-created with the same name, this ID is different.
142+
output: true
143+
- name: updateTime
144+
type: String
145+
description: The time when the feed was updated.
146+
output: true

‎mmv1/templates/terraform/samples/services/dataplex/dataplex_metadata_feed_basic.tf.tmpl‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
data "google_project" "project" {}
2+
3+
resource "google_pubsub_topic" "primary" {
4+
name = "{{index $.ResourceIdVars "metadata_feed_id"}}-topic"
5+
}
6+
7+
resource "google_pubsub_topic_iam_member" "primary_publisher" {
8+
topic = google_pubsub_topic.primary.name
9+
role = "roles/pubsub.publisher"
10+
member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-dataplex.iam.gserviceaccount.com"
11+
}
12+
13+
resource "google_pubsub_topic_iam_member" "primary_viewer" {
14+
topic = google_pubsub_topic.primary.name
15+
role = "roles/pubsub.viewer"
16+
member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-dataplex.iam.gserviceaccount.com"
17+
18+
depends_on = [google_pubsub_topic_iam_member.primary_publisher]
19+
}
20+
21+
resource "google_dataplex_entry_group" "group" {
22+
entry_group_id = "{{index $.ResourceIdVars "metadata_feed_id"}}-group"
23+
project = "{{index $.TestEnvVars "project_name"}}"
24+
location = "us-central1"
25+
}
26+
27+
resource "google_dataplex_entry_type" "type" {
28+
entry_type_id = "{{index $.ResourceIdVars "metadata_feed_id"}}-type"
29+
project = "{{index $.TestEnvVars "project_name"}}"
30+
location = "us-central1"
31+
}
32+
33+
resource "google_dataplex_aspect_type" "aspect" {
34+
aspect_type_id = "{{index $.ResourceIdVars "metadata_feed_id"}}-aspect"
35+
project = "{{index $.TestEnvVars "project_name"}}"
36+
location = "us-central1"
37+
38+
metadata_template = <<EOF
39+
{
40+
"name": "tf-test-template",
41+
"type": "record",
42+
"recordFields": [
43+
{
44+
"name": "type",
45+
"type": "enum",
46+
"annotations": {
47+
"displayName": "Type",
48+
"description": "Specifies the type of view represented by the entry."
49+
},
50+
"index": 1,
51+
"constraints": {
52+
"required": true
53+
},
54+
"enumValues": [
55+
{
56+
"name": "VIEW",
57+
"index": 1
58+
}
59+
]
60+
}
61+
]
62+
}
63+
EOF
64+
}
65+
66+
resource "google_dataplex_metadata_feed" "primary" {
67+
metadata_feed_id = "{{index $.ResourceIdVars "metadata_feed_id"}}"
68+
location = "us-central1"
69+
project = "{{index $.TestEnvVars "project_name"}}"
70+
pubsub_topic = google_pubsub_topic.primary.id
71+
72+
labels = {
73+
foo = "{{index $.Vars "label_val"}}"
74+
}
75+
76+
filters {
77+
aspect_types = ["projects/${data.google_project.project.number}/locations/us-central1/aspectTypes/${google_dataplex_aspect_type.aspect.aspect_type_id}"]
78+
change_types = ["CREATE"]
79+
entry_types = ["projects/${data.google_project.project.number}/locations/us-central1/entryTypes/${google_dataplex_entry_type.type.entry_type_id}"]
80+
}
81+
82+
scope {
83+
entry_groups = ["projects/${data.google_project.project.number}/locations/us-central1/entryGroups/${google_dataplex_entry_group.group.entry_group_id}"]
84+
}
85+
86+
depends_on = [
87+
google_pubsub_topic_iam_member.primary_publisher,
88+
google_pubsub_topic_iam_member.primary_viewer
89+
]
90+
}

‎mmv1/templates/terraform/samples/services/dataplex/dataplex_metadata_feed_projects.tf.tmpl‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
data "google_project" "project" {}
2+
3+
resource "google_pubsub_topic" "primary" {
4+
name = "{{index $.ResourceIdVars "metadata_feed_id"}}-topic"
5+
}
6+
7+
resource "google_pubsub_topic_iam_member" "primary_publisher" {
8+
topic = google_pubsub_topic.primary.name
9+
role = "roles/pubsub.publisher"
10+
member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-dataplex.iam.gserviceaccount.com"
11+
}
12+
13+
resource "google_pubsub_topic_iam_member" "primary_viewer" {
14+
topic = google_pubsub_topic.primary.name
15+
role = "roles/pubsub.viewer"
16+
member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-dataplex.iam.gserviceaccount.com"
17+
18+
depends_on = [google_pubsub_topic_iam_member.primary_publisher]
19+
}
20+
21+
resource "google_dataplex_entry_type" "type" {
22+
entry_type_id = "{{index $.ResourceIdVars "metadata_feed_id"}}-type"
23+
project = "{{index $.TestEnvVars "project_name"}}"
24+
location = "us-central1"
25+
}
26+
27+
resource "google_dataplex_aspect_type" "aspect" {
28+
aspect_type_id = "{{index $.ResourceIdVars "metadata_feed_id"}}-aspect"
29+
project = "{{index $.TestEnvVars "project_name"}}"
30+
location = "us-central1"
31+
32+
metadata_template = <<EOF
33+
{
34+
"name": "tf-test-template",
35+
"type": "record",
36+
"recordFields": [
37+
{
38+
"name": "type",
39+
"type": "enum",
40+
"annotations": {
41+
"displayName": "Type",
42+
"description": "Specifies the type of view represented by the entry."
43+
},
44+
"index": 1,
45+
"constraints": {
46+
"required": true
47+
},
48+
"enumValues": [
49+
{
50+
"name": "VIEW",
51+
"index": 1
52+
}
53+
]
54+
}
55+
]
56+
}
57+
EOF
58+
}
59+
60+
resource "google_dataplex_metadata_feed" "primary" {
61+
metadata_feed_id = "{{index $.ResourceIdVars "metadata_feed_id"}}"
62+
location = "us-central1"
63+
project = "{{index $.TestEnvVars "project_name"}}"
64+
pubsub_topic = google_pubsub_topic.primary.id
65+
66+
labels = {
67+
foo = "{{index $.Vars "label_val"}}"
68+
}
69+
70+
filters {
71+
aspect_types = ["projects/${data.google_project.project.number}/locations/us-central1/aspectTypes/${google_dataplex_aspect_type.aspect.aspect_type_id}"]
72+
change_types = ["CREATE"]
73+
entry_types = ["projects/${data.google_project.project.number}/locations/us-central1/entryTypes/${google_dataplex_entry_type.type.entry_type_id}"]
74+
}
75+
76+
scope {
77+
projects = ["projects/${data.google_project.project.number}"]
78+
}
79+
80+
depends_on = [
81+
google_pubsub_topic_iam_member.primary_publisher,
82+
google_pubsub_topic_iam_member.primary_viewer
83+
]
84+
}

0 commit comments

Comments
 (0)