Skip to content

Commit 65a4499

Browse files
vinaykpudvinaykpud
authored
Merge branch 'main' into feature/datafusion-dsl-query-agg
2 parents 54cc77f + b1de20f commit 65a4499

31 files changed

Lines changed: 1241 additions & 34 deletions

‎.github/workflows/publish-maven-snapshots.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,15 @@ jobs:
2525
distribution: temurin
2626
java-version: 21
2727

28+
- name: Install protoc (Linux)
29+
run: |
30+
if [ "$(uname -m)" = "x86_64" ]; then \
31+
curl -SfL https://github.com/protocolbuffers/protobuf/releases/download/v33.0/protoc-33.0-linux-x86_64.zip -o protoc.zip; \
32+
else \
33+
curl -SfL https://github.com/protocolbuffers/protobuf/releases/download/v33.0/protoc-33.0-linux-aarch_64.zip -o protoc.zip; \
34+
fi; \
35+
unzip protoc.zip -d $HOME/.local && rm -v protoc.zip && protoc --version
36+
2837
- name: Load secret
2938
uses: 1password/load-secrets-action@v3
3039
with:

‎CHANGELOG.md‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,109 @@ As of the 3.6 release [the CHANGELOG is no longer used][1] to generate release n
44

55
[1]: https://github.com/opensearch-project/OpenSearch/issues/21071
66
[2]: https://github.com/opensearch-project/OpenSearch/pulls?q=sort%3Amerged-desc+is%3Apr+-label%3Askip-changelog+is%3Amerged+base%3Amain+
7+
## [Unreleased 3.x]
8+
### Added
9+
- Add bitmap64 query support ([#20606](https://github.com/opensearch-project/OpenSearch/pull/20606))
10+
- Add ProfilingWrapper interface for plugin access to delegates in profiling decorators ([#20607](https://github.com/opensearch-project/OpenSearch/pull/20607))
11+
- Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532))
12+
- Choose the best performing node when writing with append-only index ([#20065](https://github.com/opensearch-project/OpenSearch/pull/20065))
13+
- Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463), [#20649](https://github.com/opensearch-project/OpenSearch/pull/20649))
14+
- Add range validations in query builder and field mapper ([#20497](https://github.com/opensearch-project/OpenSearch/issues/20497))
15+
- Support TLS cert hot-reload for Arrow Flight transport ([#20700](https://github.com/opensearch-project/OpenSearch/pull/20700))
16+
- [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151))
17+
- Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588))
18+
- Add mapper_settings support and field_mapping mapper type for pull-based ingestion([#20722](https://github.com/opensearch-project/OpenSearch/pull/20722))
19+
- Introduced strategy planner interfaces for indexing and deletion ([#20585](https://github.com/opensearch-project/OpenSearch/pull/20585))
20+
- Implement FieldMappingIngestionMessageMapper for pull-based ingestion ([#20729](https://github.com/opensearch-project/OpenSearch/pull/20729))
21+
- Added support of WarmerRefreshListener in NRTReplicationEngine to trigger warmer after replication on replica shards ([#20650](https://github.com/opensearch-project/OpenSearch/pull/20650))
22+
- WLM group custom search settings - groundwork and timeout ([#20536](https://github.com/opensearch-project/OpenSearch/issues/20536))
23+
- Add ingest pipeline support for pull-based ingestion ([#20873](https://github.com/opensearch-project/OpenSearch/issues/20873))
24+
- Expose JVM runtime metrics via telemetry framework ([#20844](https://github.com/opensearch-project/OpenSearch/pull/20844))
25+
- Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503))
26+
- Add new setting property 'Sensitive' for tiering dynamic settings ([#20901](https://github.com/opensearch-project/OpenSearch/pull/20901))
27+
- Add ref_path support for package-based hunspell dictionary loading ([#20840](https://github.com/opensearch-project/OpenSearch/pull/20840))
28+
- Add support for enabling pluggable data formats, starting with phase-1 of decoupling shard from engine, and introducing basic abstractions ([#20675](https://github.com/opensearch-project/OpenSearch/pull/20675))
29+
- Add concurrent queue in libs and composite engine sandbox plugin ([#20909](https://github.com/opensearch-project/OpenSearch/pull/20909))
30+
- Add interface for the Multi format merge flow ([#20908](https://github.com/opensearch-project/OpenSearch/pull/20908))
31+
- Add CatalogSnapshotManager lifecycle management with reference-counted snapshot tracking and serialization support for Segment and WriterFileSet ([#20982](https://github.com/opensearch-project/OpenSearch/pull/20982))
32+
33+
- Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526))
34+
- Add a new static method to IndicesOptions API to expose `STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED` index option ([#20980](https://github.com/opensearch-project/OpenSearch/pull/20980))
35+
- Add changes for parquet only query with datafusion ([#21057](https://github.com/opensearch-project/OpenSearch/pull/21057))
36+
- Add skeleton structure for tiering status API ([#21017](https://github.com/opensearch-project/OpenSearch/pull/21017))
37+
38+
### Changed
39+
- Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
40+
- Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570))
41+
- Use env variable (OPENSEARCH_FIPS_MODE) to enable opensearch to run in FIPS enforced mode instead of checking for existence of bcFIPS jars ([#20625](https://github.com/opensearch-project/OpenSearch/pull/20625))
42+
- Update streaming flag to use search request context ([#20530](https://github.com/opensearch-project/OpenSearch/pull/20530))
43+
- Move pull-based ingestion classes from experimental to publicAPI ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704))
44+
- Lazy init stored field reader in SourceLookup ([#20827](https://github.com/opensearch-project/OpenSearch/pull/20827))
45+
* Improved error message when trying to open an index originally created with Elasticsearch on OpenSearch ([#20512](https://github.com/opensearch-project/OpenSearch/pull/20512))
46+
- Updated MMapDirectory to use ReadAdviseByContext rather than default readadvise of Lucene([#21031](https://github.com/opensearch-project/OpenSearch/pull/21031))
47+
48+
### Fixed
49+
- Relax index template pattern overlap check to use minimum-string heuristic, allowing distinguishable multi-wildcard patterns at the same priority ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702))
50+
- Fix `AutoForceMergeMetrics` silently dropping tags due to unreassigned `addTag()` return value ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))
51+
- Fix flaky test failures in ShardsLimitAllocationDeciderIT ([#20375](https://github.com/opensearch-project/OpenSearch/pull/20375))
52+
- Prevent criteria update for context aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250))
53+
- Update EncryptedBlobContainer to adhere limits while listing blobs in specific sort order if wrapped blob container supports ([#20514](https://github.com/opensearch-project/OpenSearch/pull/20514))
54+
- [segment replication] Fix segment replication infinite retry due to stale metadata checkpoint ([#20551](https://github.com/opensearch-project/OpenSearch/pull/20551))
55+
- Changing opensearch.cgroups.hierarchy.override causes java.lang.SecurityException exception ([#20565](https://github.com/opensearch-project/OpenSearch/pull/20565))
56+
- Fix CriteriaBasedCodec to work with delegate codec. ([#20442](https://github.com/opensearch-project/OpenSearch/pull/20442))
57+
- Fix WLM workload group creation failing due to updated_at clock skew ([#20486](https://github.com/opensearch-project/OpenSearch/pull/20486))
58+
- Fix copy_to functionality for geo_point fields with object/array values ([#20542](https://github.com/opensearch-project/OpenSearch/pull/20542))
59+
- Fix SLF4J component error ([#20587](https://github.com/opensearch-project/OpenSearch/pull/20587))
60+
- Service does not start on Windows with OpenJDK ([#20615](https://github.com/opensearch-project/OpenSearch/pull/20615))
61+
- Update RemoteClusterStateCleanupManager to performed batched deletions of stale ClusterMetadataManifests and address deletion timeout issues ([#20566](https://github.com/opensearch-project/OpenSearch/pull/20566))
62+
- Fix the regression of terms agg optimization at high cardinality ([#20623](https://github.com/opensearch-project/OpenSearch/pull/20623))
63+
- Leveraging segment-global ordinal mapping for efficient terms aggregation ([#20624](https://github.com/opensearch-project/OpenSearch/pull/20624))
64+
- Support Docker distribution builds for ppc64le, arm64 and s390x ([#20678](https://github.com/opensearch-project/OpenSearch/pull/20678))
65+
- Harden detection of HTTP/3 support by ensuring Quic native libraries are available for the target platform ([#20680](https://github.com/opensearch-project/OpenSearch/pull/20680))
66+
- Fallback to netty client if AWS Crt client is not available on the target platform / architecture ([#20698](https://github.com/opensearch-project/OpenSearch/pull/20698))
67+
- Fix ShardSearchFailure in transport-grpc ([#20641](https://github.com/opensearch-project/OpenSearch/pull/20641))
68+
- Fix TLS cert hot-reload for Arrow Flight transport ([#20732](https://github.com/opensearch-project/OpenSearch/pull/20732))
69+
- Fix misleading heap usage cancellation message in SearchBackpressureService ([#20779](https://github.com/opensearch-project/OpenSearch/pull/20779))
70+
- Fix task details JSON logs with nested JSON in metadata are not properly escaped ([#20802](https://github.com/opensearch-project/OpenSearch/pull/20802))
71+
- Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775))
72+
- Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823))
73+
- Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842))
74+
- Fix stale segment cleanup logic for remote store ([#20976](https://github.com/opensearch-project/OpenSearch/pull/20976))
75+
- Ensure that transient ThreadContext headers with propagators survive restore ([#169373](https://github.com/opensearch-project/OpenSearch/pull/20854))
76+
- Remove X-Request-Id format restrictions and make size configurable ([#21048](https://github.com/opensearch-project/OpenSearch/pull/21048))
77+
- Handle dependencies between analyzers ([#19248](https://github.com/opensearch-project/OpenSearch/pull/19248))
78+
- Restore default `shard_path_type` to FIXED for snapshot repositories ([#20643](https://github.com/opensearch-project/OpenSearch/issues/20643))
79+
- Fix `_field_caps` returning empty results and corrupted field names for `disable_objects: true` mappings ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800))
80+
- Fix race condition in PeerFinder where concurrent connection attempts could fail ([#21055](https://github.com/opensearch-project/OpenSearch/pull/21055))
81+
82+
83+
### Dependencies
84+
- Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569))
85+
- Bump `ch.qos.logback:logback-core` and `ch.qos.logback:logback-classic` from 1.5.24 to 1.5.27 ([#20525](https://github.com/opensearch-project/OpenSearch/pull/20525))
86+
- Bump `org.apache.commons:commons-text` from 1.14.0 to 1.15.0 ([#20576](https://github.com/opensearch-project/OpenSearch/pull/20576))
87+
- Bump `aws-actions/configure-aws-credentials` from 5 to 6 ([#20577](https://github.com/opensearch-project/OpenSearch/pull/20577))
88+
- Bump `netty` from 4.2.9.Final to 4.2.12.Final ([#20586](https://github.com/opensearch-project/OpenSearch/pull/20586), [#20997](https://github.com/opensearch-project/OpenSearch/pull/20997), [#20998](https://github.com/opensearch-project/OpenSearch/pull/20998))
89+
- Bump Apache Lucene from 10.3.2 to 10.4.0 ([#20735](https://github.com/opensearch-project/OpenSearch/pull/20735))
90+
- Bump `reactor-netty` from 1.3.2 to 1.3.4 ([#20589](https://github.com/opensearch-project/OpenSearch/pull/20589), [#20834](https://github.com/opensearch-project/OpenSearch/pull/20834))
91+
- Bump `reactor` from 3.8.2 to 3.8.4 ([#20589](https://github.com/opensearch-project/OpenSearch/pull/20589), [#20834](https://github.com/opensearch-project/OpenSearch/pull/20834))
92+
- Bump `org.jruby.jcodings:jcodings` from 1.0.63 to 1.0.64 ([#20713](https://github.com/opensearch-project/OpenSearch/pull/20713))
93+
- Bump `org.jruby.joni:joni` from 2.2.3 to 2.2.7 ([#20714](https://github.com/opensearch-project/OpenSearch/pull/20714), [#20759](https://github.com/opensearch-project/OpenSearch/pull/20759))
94+
- Bump `tj-actions/changed-files` from 47.0.1 to 47.0.5 ([#20638](https://github.com/opensearch-project/OpenSearch/pull/20638), [#20716](https://github.com/opensearch-project/OpenSearch/pull/20716), [#20801](https://github.com/opensearch-project/OpenSearch/pull/20801))
95+
- Bump `com.nimbusds:nimbus-jose-jwt` from 10.7 to 10.8 ([#20715](https://github.com/opensearch-project/OpenSearch/pull/20715))
96+
- Bump OpenTelemetry to 1.60.1 and OpenTelemetry Semconv to 1.40.0 ([#20737](https://github.com/opensearch-project/OpenSearch/pull/20737), [#20797](https://github.com/opensearch-project/OpenSearch/pull/20797))
97+
- Bump `ch.qos.logback:logback-classic` from 1.5.27 to 1.5.32 ([#20761](https://github.com/opensearch-project/OpenSearch/pull/20761))
98+
- Bump `actions/upload-artifact` from 6 to 7 ([#20762](https://github.com/opensearch-project/OpenSearch/pull/20762))
99+
- Bump `org.tukaani:xz` from 1.11 to 1.12 ([#20760](https://github.com/opensearch-project/OpenSearch/pull/20760))
100+
- Bump `org.jline:jline` from 3.30.6 to 4.0.0 ([#20800](https://github.com/opensearch-project/OpenSearch/pull/20800))
101+
- Bump `com.netflix.nebula.ospackage-base` from 12.2.0 to 12.3.0 ([#20799](https://github.com/opensearch-project/OpenSearch/pull/20799))
102+
- Bump `com.netflix.nebula:gradle-info-plugin` to 16.2.1 ([#20825](https://github.com/opensearch-project/OpenSearch/pull/20825))
103+
- Bump `org.apache.logging.log4j:log4j-core` from 2.25.3 to 2.25.4 ([#21040](https://github.com/opensearch-project/OpenSearch/pull/21040))
104+
- Bump `com.sun.xml.bind:jaxb-impl` from 4.0.6 to 4.0.7 ([#20886](https://github.com/opensearch-project/OpenSearch/pull/20886))
105+
- Bump `lycheeverse/lychee-action` from 2.7.0 to 2.8.0 ([#20888](https://github.com/opensearch-project/OpenSearch/pull/20888))
106+
- Update Jackson to 2.21.2 ([#20989](https://github.com/opensearch-project/OpenSearch/pull/20989))
107+
- Bump `org.apache.zookeeper:zookeeper` from 3.9.4 to 3.9.5 ([#20887](https://github.com/opensearch-project/OpenSearch/pull/20887))
108+
- Bump `actions/github-script` from 7 to 8 ([#21042](https://github.com/opensearch-project/OpenSearch/pull/21042))
109+
110+
### Removed
111+
112+
[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.7...main

‎DEVELOPER_GUIDE.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
- [Install Prerequisites](#install-prerequisites)
55
- [JDK](#jdk)
66
- [Custom Runtime JDK](#custom-runtime-jdk)
7+
- [Rust and Protoc](#rust-and-protoc)
78
- [Windows](#windows)
89
- [Docker](#docker)
910
- [Build](#build)
@@ -92,6 +93,20 @@ By default, the test tasks use bundled JDK runtime, configured in version catalo
9293

9394
Other kind of test tasks (integration, cluster, etc.) use the same runtime as `JAVA_HOME`. However, the build also supports compiling with one version of JDK, and testing on a different version. To do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of another JDK installation, e.g. `RUNTIME_JAVA_HOME=/usr/lib/jvm/jdk-14`. Alternatively, the runtime JDK version could be provided as the command line argument, using combination of `runtime.java=<major JDK version>` property and `JAVA<major JDK version>_HOME` environment variable, for example `./gradlew -Druntime.java=17 ...` (in this case, the tooling expects `JAVA17_HOME` environment variable to be set).
9495

96+
#### Rust and Protoc
97+
98+
Sandbox plugins such as `analytics-backend-datafusion` include a native Rust component that is compiled via Cargo during the Gradle build. Building the full project (including sandbox) requires:
99+
100+
1. **Rust toolchain**: Install via [rustup](https://rustup.rs/):
101+
```
102+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y
103+
```
104+
105+
2. **Protocol Buffers compiler (`protoc`)**: Required by the [Substrait](https://substrait.io/) dependency used in DataFusion / analytics plugins.
106+
- macOS: `brew install protobuf`
107+
- Ubuntu/Debian: `apt-get install -y protobuf-compiler`
108+
- Or download from [protobuf releases](https://github.com/protocolbuffers/protobuf/releases)
109+
95110
#### Windows
96111

97112
On Windows, set `_JAVA_OPTIONS: -Xmx4096M`. You may also need to set `LongPathsEnabled=0x1` under `Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem`.

‎gradle/libs.versions.toml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,24 @@ roaringbitmap = { group = "org.roaringbitmap", name = "RoaringBitmap", version.r
171171
spatial4j = { group = "org.locationtech.spatial4j", name = "spatial4j", version.ref = "spatial4j" }
172172
tdigest = { group = "com.tdunning", name = "t-digest", version.ref = "tdigest" }
173173

174+
# OpenTelemetry libraries
175+
opentelemetry-api = { group = "io.opentelemetry", name = "opentelemetry-api", version.ref = "opentelemetry" }
176+
opentelemetry-common = { group = "io.opentelemetry", name = "opentelemetry-common", version.ref = "opentelemetry" }
177+
opentelemetry-context = { group = "io.opentelemetry", name = "opentelemetry-context", version.ref = "opentelemetry" }
178+
opentelemetry-sdk = { group = "io.opentelemetry", name = "opentelemetry-sdk", version.ref = "opentelemetry" }
179+
opentelemetry-sdk-common = { group = "io.opentelemetry", name = "opentelemetry-sdk-common", version.ref = "opentelemetry" }
180+
opentelemetry-sdk-trace = { group = "io.opentelemetry", name = "opentelemetry-sdk-trace", version.ref = "opentelemetry" }
181+
opentelemetry-sdk-metrics = { group = "io.opentelemetry", name = "opentelemetry-sdk-metrics", version.ref = "opentelemetry" }
182+
opentelemetry-sdk-logs = { group = "io.opentelemetry", name = "opentelemetry-sdk-logs", version.ref = "opentelemetry" }
183+
opentelemetry-exporter-logging = { group = "io.opentelemetry", name = "opentelemetry-exporter-logging", version.ref = "opentelemetry" }
184+
opentelemetry-exporter-otlp = { group = "io.opentelemetry", name = "opentelemetry-exporter-otlp", version.ref = "opentelemetry" }
185+
opentelemetry-exporter-common = { group = "io.opentelemetry", name = "opentelemetry-exporter-common", version.ref = "opentelemetry" }
186+
opentelemetry-exporter-otlp-common = { group = "io.opentelemetry", name = "opentelemetry-exporter-otlp-common", version.ref = "opentelemetry" }
187+
opentelemetry-exporter-sender-okhttp = { group = "io.opentelemetry", name = "opentelemetry-exporter-sender-okhttp", version.ref = "opentelemetry" }
188+
opentelemetry-semconv = { group = "io.opentelemetry.semconv", name = "opentelemetry-semconv", version.ref = "opentelemetrysemconv" }
189+
opentelemetry-sdk-testing = { group = "io.opentelemetry", name = "opentelemetry-sdk-testing", version.ref = "opentelemetry" }
190+
kotlin-stdlib = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib", version.ref = "kotlin" }
191+
174192
[bundles]
175193
asm = [
176194
"asm-analysis",
@@ -224,3 +242,19 @@ bouncycastle = [
224242
"bouncycastle-pg",
225243
"bouncycastle-util"
226244
]
245+
246+
opentelemetry = [
247+
"opentelemetry-api",
248+
"opentelemetry-common",
249+
"opentelemetry-context",
250+
"opentelemetry-sdk",
251+
"opentelemetry-sdk-common",
252+
"opentelemetry-sdk-trace",
253+
"opentelemetry-sdk-metrics",
254+
"opentelemetry-sdk-logs",
255+
"opentelemetry-exporter-logging",
256+
"opentelemetry-exporter-otlp",
257+
"opentelemetry-exporter-common",
258+
"opentelemetry-exporter-otlp-common",
259+
"opentelemetry-semconv"
260+
]

0 commit comments

Comments
 (0)