Skip to content

Commit d582cc1

Browse files
committed
Merge remote-tracking branch 'upstream/main' into ISSUE-27207
2 parents fe42ac1 + f69d753 commit d582cc1

200 files changed

Lines changed: 4856 additions & 1312 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/integration-tests-mysql-elasticsearch.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
name: Integration Tests - MySQL + Elasticsearch
1313

1414
on:
15+
merge_group:
1516
workflow_dispatch:
1617
push:
1718
branches:
@@ -96,7 +97,7 @@ jobs:
9697
- name: Checkout
9798
uses: actions/checkout@v4
9899
with:
99-
ref: ${{ github.event.pull_request.head.sha }}
100+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
100101

101102
- name: Cache Maven dependencies
102103
id: cache-output

.github/workflows/integration-tests-postgres-opensearch.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
name: Integration Tests - PostgreSQL + OpenSearch
1313

1414
on:
15+
merge_group:
1516
workflow_dispatch:
1617
push:
1718
branches:
@@ -96,7 +97,7 @@ jobs:
9697
- name: Checkout
9798
uses: actions/checkout@v4
9899
with:
99-
ref: ${{ github.event.pull_request.head.sha }}
100+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
100101

101102
- name: Cache Maven dependencies
102103
id: cache-output

.github/workflows/java-checkstyle.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
name: Java Checkstyle
1313

1414
on:
15+
merge_group:
1516
# Trigger analysis when pushing in master or pull requests, and when creating
1617
# a pull request.
1718
push:
@@ -68,7 +69,7 @@ jobs:
6869
- name: Checkout
6970
uses: actions/checkout@v4
7071
with:
71-
ref: ${{ github.event.pull_request.head.sha }}
72+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
7273

7374
- name: Set up JDK 21
7475
uses: actions/setup-java@v4
@@ -86,7 +87,7 @@ jobs:
8687
git diff-files --quiet
8788
8889
- name: Create a comment in the PR with the instructions
89-
if: steps.git.outcome != 'success'
90+
if: ${{ steps.git.outcome != 'success' && github.event_name == 'pull_request_target' }}
9091
uses: peter-evans/create-or-update-comment@v1
9192
with:
9293
issue-number: ${{ github.event.pull_request.number }}

.github/workflows/openmetadata-service-unit-tests.yml

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
name: OpenMetadata Service Unit Tests
1313

1414
on:
15+
merge_group:
1516
workflow_dispatch:
1617
push:
1718
branches:
@@ -39,6 +40,7 @@ jobs:
3940
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
4041
outputs:
4142
java: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.java }}
43+
k8s_operator: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.k8s_operator }}
4244
steps:
4345
- name: Checkout
4446
uses: actions/checkout@v4
@@ -59,6 +61,8 @@ jobs:
5961
- 'pom.xml'
6062
- 'Makefile'
6163
- 'bootstrap/**'
64+
k8s_operator:
65+
- 'openmetadata-k8s-operator/**'
6266
6367
openmetadata-service-unit-tests:
6468
runs-on: ubuntu-latest
@@ -121,13 +125,62 @@ jobs:
121125
report_paths: "openmetadata-service/target/surefire-reports/TEST-*.xml"
122126
check_name: "Test Report (${{ matrix.database }})"
123127

128+
k8s_operator-unit-tests:
129+
runs-on: ubuntu-latest
130+
timeout-minutes: 30
131+
needs: changes
132+
if: ${{ needs.changes.outputs.k8s_operator == 'true' }}
133+
steps:
134+
- name: Checkout
135+
uses: actions/checkout@v4
136+
with:
137+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
138+
139+
- name: Cache Maven dependencies
140+
uses: actions/cache@v4
141+
with:
142+
path: ~/.m2
143+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
144+
restore-keys: |
145+
${{ runner.os }}-maven-
146+
147+
- name: Set up JDK 21
148+
uses: actions/setup-java@v4
149+
with:
150+
java-version: "21"
151+
distribution: "temurin"
152+
153+
- name: Build k8s-operator dependencies
154+
run: |
155+
mvn -B clean install -pl openmetadata-k8s-operator -am -DskipTests
156+
157+
- name: Run k8s-operator unit tests
158+
run: |
159+
mvn -B test -pl openmetadata-k8s-operator
160+
161+
- name: Upload surefire reports
162+
if: ${{ failure() && hashFiles('openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml') != '' }}
163+
uses: actions/upload-artifact@v4
164+
with:
165+
name: k8s-operator-surefire-reports
166+
path: openmetadata-k8s-operator/target/surefire-reports/
167+
168+
- name: Publish Test Report
169+
if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && hashFiles('openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml') != '' }}
170+
uses: scacap/action-surefire-report@v1
171+
with:
172+
github_token: ${{ secrets.GITHUB_TOKEN }}
173+
fail_on_test_failures: true
174+
report_paths: "openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml"
175+
check_name: "K8s Operator Test Report"
176+
124177
# Single required-check gate for branch protection.
125178
# Skipped (= "Success") when all test jobs pass or are legitimately skipped.
126179
# Runs and exits 1 only when a test job fails or is cancelled.
127180
# Set "OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status" as the sole required check for this workflow.
128181
openmetadata-service-unit-tests-status:
129182
name: openmetadata-service-unit-tests-status
130-
needs: [changes, openmetadata-service-unit-tests]
183+
needs: [changes, openmetadata-service-unit-tests, k8s_operator-unit-tests]
131184
if: ${{ failure() || cancelled() }}
132185
runs-on: ubuntu-latest
133186
steps:

.github/workflows/playwright-postgresql-e2e.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
name: Postgresql PR Playwright E2E Tests
1616
on:
17+
merge_group:
1718
workflow_dispatch:
1819
pull_request_target:
1920
types:
@@ -66,7 +67,7 @@ jobs:
6667
- name: Checkout
6768
uses: actions/checkout@v4
6869
with:
69-
ref: ${{ github.event.pull_request.head.sha }}
70+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
7071

7172
- name: Setup JDK 21
7273
uses: actions/setup-java@v4
@@ -120,7 +121,7 @@ jobs:
120121
- name: Checkout
121122
uses: actions/checkout@v4
122123
with:
123-
ref: ${{ github.event.pull_request.head.sha }}
124+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
124125

125126
- name: Download Maven build artifact
126127
uses: actions/download-artifact@v4

.github/workflows/py-checkstyle.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ name: Python Checkstyle
1414
# read-write repo token
1515
# access to secrets
1616
on:
17+
merge_group:
1718
pull_request_target:
1819
types: [labeled, opened, synchronize, reopened, ready_for_review]
1920
paths-ignore:
@@ -56,7 +57,7 @@ jobs:
5657
- name: Checkout
5758
uses: actions/checkout@v4
5859
with:
59-
ref: ${{ github.event.pull_request.head.sha }}
60+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
6061

6162
- name: Set up Python 3.10
6263
uses: actions/setup-python@v5
@@ -84,7 +85,7 @@ jobs:
8485
make py_format_check
8586
8687
- name: Create a comment in the PR with the instructions
87-
if: steps.style.outcome != 'success'
88+
if: ${{ steps.style.outcome != 'success' && github.event_name == 'pull_request_target' }}
8889
uses: peter-evans/create-or-update-comment@v1
8990
with:
9091
issue-number: ${{ github.event.pull_request.number }}

.github/workflows/py-tests-postgres.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
name: py-tests-postgres
1313
on:
14+
merge_group:
1415
workflow_dispatch:
1516
pull_request_target:
1617
types: [labeled, opened, synchronize, reopened, ready_for_review]
@@ -103,7 +104,7 @@ jobs:
103104
- name: Checkout
104105
uses: actions/checkout@v4
105106
with:
106-
ref: ${{ github.event.pull_request.head.sha }}
107+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
107108

108109
- name: Setup Openmetadata Test Environment
109110
uses: ./.github/actions/setup-openmetadata-test-environment

.github/workflows/py-tests.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
name: py-tests
1313
on:
14+
merge_group:
1415
workflow_dispatch:
1516
pull_request_target:
1617
types: [labeled, opened, synchronize, reopened, ready_for_review]
@@ -89,7 +90,7 @@ jobs:
8990
- name: Checkout
9091
uses: actions/checkout@v4
9192
with:
92-
ref: ${{ github.event.pull_request.head.sha }}
93+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
9394

9495
- name: Setup Openmetadata Test Environment
9596
uses: ./.github/actions/setup-openmetadata-test-environment
@@ -177,7 +178,7 @@ jobs:
177178
- name: Checkout
178179
uses: actions/checkout@v4
179180
with:
180-
ref: ${{ github.event.pull_request.head.sha }}
181+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
181182

182183
- name: Setup Openmetadata Test Environment
183184
uses: ./.github/actions/setup-openmetadata-test-environment
@@ -230,7 +231,7 @@ jobs:
230231
- name: Checkout
231232
uses: actions/checkout@v4
232233
with:
233-
ref: ${{ github.event.pull_request.head.sha }}
234+
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
234235
fetch-depth: 0
235236
filter: blob:none
236237

.github/workflows/security-scan.yml

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,142 @@ on:
1616
workflow_dispatch:
1717

1818
jobs:
19+
vulnerability-scan:
20+
runs-on: ubuntu-latest
21+
environment: security-scan
22+
permissions:
23+
contents: read
24+
steps:
25+
- uses: actions/checkout@v4
26+
27+
- name: Setup Node.js
28+
uses: actions/setup-node@v4
29+
with:
30+
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
31+
32+
- name: Enable yarn
33+
run: corepack enable
34+
35+
- name: Install UI dependencies
36+
working-directory: openmetadata-ui/src/main/resources/ui
37+
run: yarn install --frozen-lockfile --ignore-scripts
38+
39+
- name: Run Retire.js scan
40+
id: retire-scan
41+
continue-on-error: true
42+
working-directory: openmetadata-ui/src/main/resources/ui
43+
run: |
44+
npx retire@5 \
45+
--path node_modules/ \
46+
--severity medium \
47+
--outputformat json \
48+
--outputpath retire-report.json
49+
50+
- name: Verify report was generated
51+
working-directory: openmetadata-ui/src/main/resources/ui
52+
run: |
53+
if [ ! -f retire-report.json ]; then
54+
echo '::error::retire-report.json was not generated — retire scan may have crashed'
55+
exit 1
56+
fi
57+
58+
- name: Upload Retire.js Report
59+
if: success()
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: retire-js-report
63+
path: openmetadata-ui/src/main/resources/ui/retire-report.json
64+
retention-days: 30
65+
66+
- name: Publish Retire.js Summary
67+
if: success()
68+
working-directory: openmetadata-ui/src/main/resources/ui
69+
run: |
70+
python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY
71+
import json
72+
73+
SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
74+
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
75+
NM = "node_modules/"
76+
77+
def escape(text):
78+
return str(text).replace('|', '\\|').replace('`', "'")
79+
80+
try:
81+
with open("retire-report.json") as f:
82+
data = json.load(f)
83+
except FileNotFoundError:
84+
print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.")
85+
raise SystemExit(0)
86+
87+
findings = data.get("data", [])
88+
libs = {}
89+
for item in findings:
90+
filepath = item.get("file", "")
91+
short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath
92+
for result in item.get("results", []):
93+
key = (result.get("component", ""), result.get("version", ""))
94+
if key not in libs:
95+
libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])}
96+
if short not in libs[key]["files"]:
97+
libs[key]["files"].append(short)
98+
99+
print("## Retire.js Scan Results\n")
100+
101+
if not libs:
102+
print("✅ No vulnerable libraries found.")
103+
else:
104+
total_vulns = sum(len(v["vulns"]) for v in libs.values())
105+
print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n")
106+
107+
for (component, version), info in sorted(libs.items(), key=lambda x: min(
108+
(SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)):
109+
top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3))
110+
icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪")
111+
print(f"### {icon} {component} {version}\n")
112+
print("| Severity | CVE | Summary |")
113+
print("|---|---|---|")
114+
for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)):
115+
sev = vuln.get("severity", "")
116+
ids = vuln.get("identifiers", {})
117+
cves = ids.get("CVE", [])
118+
summary = ids.get("summary", "").split("\n")[0][:120]
119+
cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—")
120+
print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |")
121+
print("\n**Bundled in:**")
122+
for f in info["files"]:
123+
print(f"- `{f}`")
124+
print()
125+
EOF
126+
127+
- name: Slack on Failure
128+
if: steps.retire-scan.outcome == 'failure'
129+
uses: slackapi/slack-github-action@v1.23.0
130+
with:
131+
channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
132+
payload: |
133+
{
134+
"text": "🚨 Vulnerability scan failed, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>. 🚨"
135+
}
136+
env:
137+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
138+
139+
- name: Slack on Success
140+
if: steps.retire-scan.outcome == 'success'
141+
uses: slackapi/slack-github-action@v1.23.0
142+
with:
143+
channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
144+
payload: |
145+
{
146+
"text": "🟢 Vulnerability scan passed for OpenMetadata Repo, please check it <https://github.com/open-metadata/OpenMetadata/actions/runs/${{ github.run_id }}|here>."
147+
}
148+
env:
149+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
150+
151+
- name: Force failure on vulnerabilities found
152+
if: steps.retire-scan.outcome == 'failure'
153+
run: exit 1
154+
19155
security-scan:
20156
runs-on: ubuntu-latest
21157
environment: security-scan

0 commit comments

Comments
 (0)