Skip to content

Commit f382d9b

Browse files
simonelbazsimonelbaz
authored
Merge branch 'opensearch-project:main' into dlqpathstyle
2 parents 81c7d59 + 8632f39 commit f382d9b

752 files changed

Lines changed: 14143 additions & 1414 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
2-
* @sb2k16 @chenqi0805 @engechas @san81 @srikanthjg @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh
2+
* @sb2k16 @engechas @san81 @srikanthjg @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh

‎.github/scripts/check-license-headers.py‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env python3
2+
3+
#
4+
# Copyright OpenSearch Contributors
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# The OpenSearch Contributors require contributions made to
8+
# this file be licensed under the Apache-2.0 license or a
9+
# compatible open source license.
10+
#
11+
12+
"""
13+
License Header Compliance Checker for OpenSearch Data Prepper
14+
15+
This script checks that files contain the required license headers
16+
as specified in CONTRIBUTING.md.
17+
18+
Usage:
19+
python check-license-headers.py file1.java file2.py ...
20+
echo "file1.java\nfile2.py" | python check-license-headers.py
21+
"""
22+
23+
import os
24+
import sys
25+
from pathlib import Path
26+
from typing import List
27+
28+
# File extensions that require license headers
29+
SUPPORTED_EXTENSIONS = {
30+
'.java', '.groovy', '.gradle', # Java ecosystem
31+
'.py', # Python
32+
'.sh', '.bash', '.zsh', # Shell scripts
33+
'.yaml', '.yml', # YAML files
34+
'.properties', # Properties files
35+
}
36+
37+
def needs_license_header(file_path: str) -> bool:
38+
"""Check if a file needs a license header based on its extension."""
39+
path = Path(file_path)
40+
return path.suffix.lower() in SUPPORTED_EXTENSIONS
41+
42+
def check_file_header(file_path: str) -> bool:
43+
"""Check if a file has the required complete license header."""
44+
if not Path(file_path).exists():
45+
return True
46+
47+
try:
48+
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
49+
# Read first 15 lines to check for license header
50+
lines = []
51+
for i, line in enumerate(f):
52+
if i >= 15: # Only check first 15 lines
53+
break
54+
lines.append(line)
55+
56+
content = ''.join(lines)
57+
58+
# Check for all 5 required license header components
59+
required_components = [
60+
'Copyright OpenSearch Contributors',
61+
'SPDX-License-Identifier: Apache-2.0',
62+
'The OpenSearch Contributors require contributions made to',
63+
'this file be licensed under the Apache-2.0 license or a',
64+
'compatible open source license.'
65+
]
66+
67+
# All components must be present
68+
for component in required_components:
69+
if component not in content:
70+
return False
71+
72+
return True
73+
74+
except Exception as e:
75+
print(f"Error reading file {file_path}: {e}", file=sys.stderr)
76+
return True # Skip files we can't read
77+
78+
def get_files_to_check() -> List[str]:
79+
"""Get files to check from command line args or stdin."""
80+
if len(sys.argv) > 1:
81+
# Files provided as command line arguments
82+
return sys.argv[1:]
83+
else:
84+
# Read files from stdin
85+
files = []
86+
for line in sys.stdin:
87+
file_path = line.strip()
88+
if file_path:
89+
files.append(file_path)
90+
return files
91+
92+
def main():
93+
"""Main function to check license headers."""
94+
files_to_check = get_files_to_check()
95+
96+
if not files_to_check:
97+
print("No files to check", file=sys.stderr)
98+
return
99+
100+
print(f"Checking {len(files_to_check)} files for license headers.")
101+
102+
violations = []
103+
104+
for file_path in files_to_check:
105+
print(f"Checking: {file_path}")
106+
107+
if not Path(file_path).exists():
108+
print(f" File not found: {file_path}")
109+
continue
110+
111+
# Skip if doesn't need header
112+
if not needs_license_header(file_path):
113+
print(f" Skipped (no header needed): {file_path}")
114+
continue
115+
116+
# Check header
117+
if not check_file_header(file_path):
118+
violations.append(f"- `{file_path}`")
119+
print(f" ❌ Missing license header: {file_path}")
120+
else:
121+
print(f" ✅ Header OK: {file_path}")
122+
123+
# Output results
124+
if violations:
125+
print(f"\n❌ Found {len(violations)} license header violations:")
126+
127+
violation_text = '\n'.join(violations)
128+
129+
# Set output for GitHub Actions
130+
github_output = os.environ.get('GITHUB_OUTPUT')
131+
if github_output:
132+
with open(github_output, 'a') as f:
133+
f.write(f"violations<<EOF\n{violation_text}\nEOF\n")
134+
135+
print("\nViolations:")
136+
for violation in violations:
137+
print(f" {violation}")
138+
139+
sys.exit(1)
140+
else:
141+
print("\n✅ All files have proper license headers!")
142+
# Set empty output for GitHub Actions
143+
github_output = os.environ.get('GITHUB_OUTPUT')
144+
if github_output:
145+
with open(github_output, 'a') as f:
146+
f.write("violations=\n")
147+
148+
if __name__ == "__main__":
149+
main()

‎.github/scripts/get-new-files.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env python3
2+
3+
#
4+
# Copyright OpenSearch Contributors
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# The OpenSearch Contributors require contributions made to
8+
# this file be licensed under the Apache-2.0 license or a
9+
# compatible open source license.
10+
#
11+
12+
"""
13+
Get newly added files from Git.
14+
15+
This script identifies files added in the current PR and outputs them
16+
one per line to stdout.
17+
"""
18+
19+
import os
20+
import subprocess
21+
import sys
22+
23+
def get_newly_added_files():
24+
"""Get list of files added in this PR."""
25+
try:
26+
# Get the base branch (usually main)
27+
base_ref = os.environ.get('GITHUB_BASE_REF', 'main')
28+
29+
# Get added files in this PR
30+
result = subprocess.run([
31+
'git', 'diff', '--name-only', '--diff-filter=A',
32+
f'origin/{base_ref}...HEAD'
33+
], capture_output=True, text=True, check=True)
34+
35+
files = [f.strip() for f in result.stdout.split('\n') if f.strip()]
36+
return files
37+
38+
except subprocess.CalledProcessError as e:
39+
print(f"Error getting changed files: {e}", file=sys.stderr)
40+
return []
41+
42+
def main():
43+
"""Main function to get newly added files."""
44+
files = get_newly_added_files()
45+
46+
if not files:
47+
print("No newly added files found", file=sys.stderr)
48+
sys.exit(0)
49+
50+
# Output files one per line
51+
for file_path in files:
52+
print(file_path)
53+
54+
if __name__ == "__main__":
55+
main()

.github/workflows/data-prepper-peer-forwarder-local-node-e2e-tests.yml renamed to .github/workflows/data-prepper-kafka-backward-compatibility-e2e-tests.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This workflow will build a Java project with Gradle
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
33

4-
name: Data Prepper Local Node Peer Forwarder End-to-end test with Gradle
4+
name: Data Prepper Kafka Backward Compatibility End-to-end test with Gradle
55

66
on:
77
push:
@@ -25,5 +25,5 @@ jobs:
2525
java-version: 11
2626
- name: Checkout Data Prepper
2727
uses: actions/checkout@v2
28-
- name: Run raw-span latest release compatibility end-to-end tests with Gradle
29-
run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:peerforwarder:localAggregateEndToEndTest
28+
- name: Run Kafka backward compatibility end-to-end tests with Gradle
29+
run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:kafka-buffer-backward-compatibility:kafkaBufferBackwardCompatibilityTest

.github/workflows/data-prepper-log-analytics-basic-grok-e2e-tests.yml renamed to .github/workflows/e2e-tests-log-analytics.yml

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This workflow will build a Java project with Gradle
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
33

4-
name: Data Prepper Log Analytics Basic Grok End-to-end test with Gradle
4+
name: End-to-End Tests - Log Analytics
55

66
on:
77
push:
@@ -15,16 +15,18 @@ jobs:
1515
matrix:
1616
java: [11, 17, 21, docker]
1717
test: ['basicLogEndToEndTest', 'parallelGrokStringSubstituteTest']
18+
runner: [ubuntu-latest, ubuntu-24.04-arm]
1819
fail-fast: false
1920

20-
runs-on: ubuntu-latest
21+
runs-on: ${{ matrix.runner }}
2122

2223
steps:
23-
- name: Set up JDK 11
24-
uses: actions/setup-java@v1
24+
- name: Set up JDK
25+
uses: actions/setup-java@v2
2526
with:
27+
distribution: 'temurin'
2628
java-version: 11
2729
- name: Checkout Data Prepper
2830
uses: actions/checkout@v2
29-
- name: Run basic grok end-to-end tests with Gradle
31+
- name: Run log analytics end-to-end tests with Gradle
3032
run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }}

.github/workflows/data-prepper-peer-forwarder-static-e2e-tests.yml renamed to .github/workflows/e2e-tests-peer-forwarder.yml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This workflow will build a Java project with Gradle
22
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
33

4-
name: Data Prepper Static List Peer Forwarder with SSL and mTLS End-to-end test with Gradle
4+
name: End-to-End Tests - Peer Forwarder
55

66
on:
77
push:
@@ -14,17 +14,19 @@ jobs:
1414
strategy:
1515
matrix:
1616
java: [11, 17, 21, docker]
17-
test: ['staticAggregateEndToEndTest', 'staticLogMetricsEndToEndTest']
17+
test: ['staticAggregateEndToEndTest', 'staticLogMetricsEndToEndTest', 'localAggregateEndToEndTest']
18+
runner: [ubuntu-latest, ubuntu-24.04-arm]
1819
fail-fast: false
1920

20-
runs-on: ubuntu-latest
21+
runs-on: ${{ matrix.runner }}
2122

2223
steps:
23-
- name: Set up JDK 11
24-
uses: actions/setup-java@v1
24+
- name: Set up JDK
25+
uses: actions/setup-java@v2
2526
with:
27+
distribution: 'temurin'
2628
java-version: 11
2729
- name: Checkout Data Prepper
2830
uses: actions/checkout@v2
29-
- name: Run raw-span latest release compatibility end-to-end tests with Gradle
31+
- name: Run peer-forward end-to-end tests with Gradle
3032
run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:peerforwarder:${{ matrix.test }}

0 commit comments

Comments
 (0)