Skip to content

Commit 47e3644

Browse files
sesame0224sesame0224
authored
Merge branch 'numaproj:main' into feat/second-nic-communication
2 parents dd7c203 + 6cbe3a7 commit 47e3644

332 files changed

Lines changed: 28180 additions & 7750 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959
unit-tests:
6060
name: Unit Tests
6161
runs-on: ubuntu-24.04
62-
timeout-minutes: 15
62+
timeout-minutes: 20
6363
steps:
6464
- name: Start JetStream
6565
run: docker run -d -p 4222:4222 nats:latest -js
@@ -122,6 +122,11 @@ jobs:
122122
run: |
123123
cargo fmt -- --check
124124
125+
- name: Check Clippy
126+
working-directory: ./rust
127+
run: |
128+
cargo clippy-ci
129+
125130
- name: Upload coverage reports to Codecov
126131
uses: codecov/codecov-action@v4
127132
with:

‎CHANGELOG.md‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,57 @@
11
# Changelog
22

3+
## v1.7.4 (2026-03-13)
4+
5+
* [be04ae80](https://github.com/numaproj/numaflow/commit/be04ae805ed4a9742e9be1711e6271fc12548c1f) Update manifests to v1.7.4
6+
7+
### Contributors
8+
9+
* Yashash Lokesh
10+
11+
## v1.7.3 (2026-02-22)
12+
13+
* [90a31118](https://github.com/numaproj/numaflow/commit/90a31118d901a7ce067d5f33774a33b942fa325f) Update manifests to v1.7.3
14+
* [9252e08a](https://github.com/numaproj/numaflow/commit/9252e08ad7d2d9e67036f9dfa6050f0cdceb0554) fix: sum of rate metrics across all partitions (#3243)
15+
16+
### Contributors
17+
18+
* Yashash
19+
* Yashash Lokesh
20+
21+
## v1.7.2 (2026-02-13)
22+
23+
* [f107d3b3](https://github.com/numaproj/numaflow/commit/f107d3b30e2a5f9161daf1942dbf3a8fdec903fd) Update manifests to v1.7.2
24+
* [a072ad0b](https://github.com/numaproj/numaflow/commit/a072ad0b75cb50507c55111a989d8a540ca35197) fix(UI): pending chart for source vertex (#3209)
25+
* [7f2c9890](https://github.com/numaproj/numaflow/commit/7f2c989012b828e99a6e42fe94011a5bfde7ceb7) feat: check initContainerStatuses are in Waiting or Terminated state during health check (#3226)
26+
* [1a679c72](https://github.com/numaproj/numaflow/commit/1a679c72adafa010f0618262c30e21674a524942) fix: numa container hangs occassionally when map panics
27+
* [9f9b9c60](https://github.com/numaproj/numaflow/commit/9f9b9c6041a88645e6af8f82f73c78591e395661) feat: use initContainerStatuses in health checks (#3201)
28+
* [e8dfb2fe](https://github.com/numaproj/numaflow/commit/e8dfb2fe602e01d7ff17f2a8b1360c7fbeba192f) feat: SQS System Attributes and Custom Attributes Propagation (#3095)
29+
* [9f875b2b](https://github.com/numaproj/numaflow/commit/9f875b2bd69af1331c9a1a1d15aea4c242775b2d) fix: Restart on non-retryable kafka source ack errors (#3194)
30+
* [aec4f3a2](https://github.com/numaproj/numaflow/commit/aec4f3a2ee51893da98fdd3b333c2f4c6af256a4) fix: use vertex name as partition label for source pending metric (#3205)
31+
32+
### Contributors
33+
34+
* Adarsh Jain
35+
* Dillen Padhiar
36+
* Mikael Sundberg
37+
* Sreekanth
38+
* Yashash
39+
* shrivardhan
40+
41+
## v1.7.1 (2026-01-17)
42+
43+
* [7c8b4926](https://github.com/numaproj/numaflow/commit/7c8b4926cbd39066136c7d95c9862e21ddaa5095) Update manifests to v1.7.1
44+
* [b72b8f4f](https://github.com/numaproj/numaflow/commit/b72b8f4f63b2f546c33efa969bbace8fb60546b0) fix: use std context (#3081)
45+
* [f713f246](https://github.com/numaproj/numaflow/commit/f713f246e479a55184afa82b2025492197dab8af) fix: graceful shutdown of sigkills in map udf container (#3096)
46+
* [e7b62d2e](https://github.com/numaproj/numaflow/commit/e7b62d2e7ac5788df2e2dbf6b7ce57b0cf7247f7) fix: alignment and text wrapping in Errors tab (#3078)
47+
* [b1e59968](https://github.com/numaproj/numaflow/commit/b1e599689b6edc86994e39bec7198cd15169070a) fix: add back drop metrics after sink refactor (#3074)
48+
49+
### Contributors
50+
51+
* Adarsh Jain
52+
* Vigith Maurice
53+
* Yashash Lokesh
54+
355
## v1.7.0 (2025-11-11)
456

557
* [2d68ff45](https://github.com/numaproj/numaflow/commit/2d68ff4536274a659b7282a2a65cac210f64e523) Update manifests to v1.7.0

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ RUN chmod +x /bin/entrypoint
2222
####################################################################################################
2323
# Rust binary
2424
####################################################################################################
25-
FROM lukemathwalker/cargo-chef:latest-rust-1.92 AS chef
25+
FROM lukemathwalker/cargo-chef:latest-rust-1.93 AS chef
2626
ARG TARGETPLATFORM
2727
WORKDIR /numaflow
2828
RUN apt-get update && apt-get install -y protobuf-compiler cmake clang

‎Makefile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -189,10 +189,8 @@ endif
189189
.PHONY: build-rust-in-docker
190190
build-rust-in-docker:
191191
mkdir -p dist
192-
-$(DOCKER) container ls --all --filter=ancestor='$(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)' --format "{{.ID}}" | xargs $(DOCKER) rm
193-
-$(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)
194192
DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target rust-builder -f $(DOCKERFILE) . --load
195-
export CTR=$$($(DOCKER) create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)) && $(DOCKER) cp $$CTR:/root/numaflow dist/numaflow-rs-linux-$(HOST_ARCH) && $(DOCKER) cp $$CTR:/root/entrypoint dist/entrypoint-linux-$(HOST_ARCH) && $(DOCKER) rm $$CTR && $(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)
193+
export CTR=$$($(DOCKER) create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)) && $(DOCKER) cp $$CTR:/root/numaflow dist/numaflow-rs-linux-$(HOST_ARCH) && $(DOCKER) cp $$CTR:/root/entrypoint dist/entrypoint-linux-$(HOST_ARCH)
196194

197195
.PHONY: build-rust-in-docker-multi
198196
build-rust-in-docker-multi:
@@ -316,7 +314,7 @@ docs-serve: docs
316314

317315
.PHONY: docs-linkcheck
318316
docs-linkcheck: /usr/local/bin/lychee
319-
lychee --insecure --accept '100..=399,403,429' --exclude-path=CHANGELOG.md --exclude-path=USERS.md --exclude-path=./docs/APIs.md --exclude "https://localhost:*" --exclude "http://localhost:*" --exclude "http://127.0.0.1*" --exclude "https://kubernetes.io/" *.md $(shell find ./docs -name '*.md') $(shell find ./examples -name '*.yaml')
317+
lychee --insecure --accept '100..=399,403,429' --exclude-path=CHANGELOG.md --exclude-path=USERS.md --exclude-path=./docs/APIs.md --exclude "https://localhost:*" --exclude "http://localhost:*" --exclude "http://127.0.0.1*" --exclude "https://kubernetes.io/" --exclude "https://goreportcard.com/" --exclude "https://classic.yarnpkg.com" *.md $(shell find ./docs -name '*.md') $(shell find ./examples -name '*.yaml')
320318

321319
# pre-push checks
322320

‎USERS.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ Please add your company name and initial use case (optional) below.
1010
6. [Seekr](https://www.seekr.com/) Numaflow coordinates multiple ML pipelines to rate and extract information from the pipeline input.
1111
7. [BEUMER Group](https://www.beumergroup.com/) - Signal processing for IoT-sensors, providing predictive maintenance solutions for the Intra-logistics sector.
1212
8. [NTT](https://www.rd.ntt/) - AI/ML pipeline for realizing Accelerator Chaining. Presented at KubeCon EU 25.
13-
9. [QualiBooth](https://www.qualibooth.com/) - Accessibility-focused software. It leverages Numaflow pipelines to aggregate real-time data related to web accessibility issues.
13+
9. [QualiBooth](https://www.qualibooth.com/) - Accessibility-focused software. It leverages Numaflow pipelines to aggregate real-time data related to web accessibility issues.
14+
10. [Starboard Maritime Intelligence](https://www.starboardintelligence.com/) - High-throughput data pipelines for processing live telemetry and sensor data with real-time enrichment and validation.
15+
11. [Playerdata](https://playerdata.com) - Help your athletes build confidence, stay healthy, and work smarter with PlayerData's world-class performance tracking technology.

‎api/json-schema/schema.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19688,6 +19688,10 @@
1968819688
"description": "NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/",
1968919689
"type": "object"
1969019690
},
19691+
"ordered": {
19692+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered",
19693+
"description": "Ordered enables ordered processing for this vertex, overrides pipeline-level setting. When enabled, messages will be processed in order based on their event time. Note: Reduce vertices ignore this setting as they are already partitioned."
19694+
},
1969119695
"partitions": {
1969219696
"description": "Number of partitions of the vertex owned buffers. It applies to udf and sink vertices only.",
1969319697
"format": "int32",
@@ -19885,6 +19889,10 @@
1988519889
"toVertexLimits": {
1988619890
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.VertexLimits"
1988719891
},
19892+
"toVertexOrdered": {
19893+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered",
19894+
"description": "Determine whether ordered processing is enabled for `to` vertex. If not provided, the default value is set to \"false\"."
19895+
},
1988819896
"toVertexPartitionCount": {
1988919897
"description": "The number of partitions of the to vertex, if not provided, the default value is set to \"1\".",
1989019898
"format": "int32",
@@ -21429,6 +21437,16 @@
2142921437
"description": "NoStore means there will be no persistence storage and there will be data loss during pod restarts. Use this option only if you do not care about correctness (e.g., approx statistics pipeline like sampling rate, etc.).",
2143021438
"type": "object"
2143121439
},
21440+
"io.numaproj.numaflow.v1alpha1.Ordered": {
21441+
"description": "Ordered defines the ordered processing configuration.",
21442+
"properties": {
21443+
"enabled": {
21444+
"description": "Enabled toggles ordered processing.",
21445+
"type": "boolean"
21446+
}
21447+
},
21448+
"type": "object"
21449+
},
2143221450
"io.numaproj.numaflow.v1alpha1.PBQStorage": {
2143321451
"description": "PBQStorage defines the persistence configuration for a vertex.",
2143421452
"properties": {
@@ -21574,6 +21592,10 @@
2157421592
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PipelineLimits",
2157521593
"description": "Limits define the limitations such as buffer read batch size for all the vertices of a pipeline, they could be overridden by each vertex's settings"
2157621594
},
21595+
"ordered": {
21596+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered",
21597+
"description": "Ordered enables order-preserving processing for the entire pipeline. When enabled, messages will be processed in their arrival order (FIFO within each partition). This can be overridden at the vertex level."
21598+
},
2157721599
"sideInputs": {
2157821600
"description": "SideInputs defines the Side Inputs of a pipeline.",
2157921601
"items": {
@@ -22994,6 +23016,10 @@
2299423016
"description": "NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/",
2299523017
"type": "object"
2299623018
},
23019+
"ordered": {
23020+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered",
23021+
"description": "Ordered enables ordered processing for this vertex, overrides pipeline-level setting. When enabled, messages will be processed in order based on their event time. Note: Reduce vertices ignore this setting as they are already partitioned."
23022+
},
2299723023
"partitions": {
2299823024
"description": "Number of partitions of the vertex owned buffers. It applies to udf and sink vertices only.",
2299923025
"format": "int32",

‎api/openapi-spec/swagger.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19696,6 +19696,10 @@
1969619696
"type": "string"
1969719697
}
1969819698
},
19699+
"ordered": {
19700+
"description": "Ordered enables ordered processing for this vertex, overrides pipeline-level setting. When enabled, messages will be processed in order based on their event time. Note: Reduce vertices ignore this setting as they are already partitioned.",
19701+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered"
19702+
},
1969919703
"partitions": {
1970019704
"description": "Number of partitions of the vertex owned buffers. It applies to udf and sink vertices only.",
1970119705
"type": "integer",
@@ -19896,6 +19900,10 @@
1989619900
"toVertexLimits": {
1989719901
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.VertexLimits"
1989819902
},
19903+
"toVertexOrdered": {
19904+
"description": "Determine whether ordered processing is enabled for `to` vertex. If not provided, the default value is set to \"false\".",
19905+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered"
19906+
},
1989919907
"toVertexPartitionCount": {
1990019908
"description": "The number of partitions of the to vertex, if not provided, the default value is set to \"1\".",
1990119909
"type": "integer",
@@ -21424,6 +21432,16 @@
2142421432
"description": "NoStore means there will be no persistence storage and there will be data loss during pod restarts. Use this option only if you do not care about correctness (e.g., approx statistics pipeline like sampling rate, etc.).",
2142521433
"type": "object"
2142621434
},
21435+
"io.numaproj.numaflow.v1alpha1.Ordered": {
21436+
"description": "Ordered defines the ordered processing configuration.",
21437+
"type": "object",
21438+
"properties": {
21439+
"enabled": {
21440+
"description": "Enabled toggles ordered processing.",
21441+
"type": "boolean"
21442+
}
21443+
}
21444+
},
2142721445
"io.numaproj.numaflow.v1alpha1.PBQStorage": {
2142821446
"description": "PBQStorage defines the persistence configuration for a vertex.",
2142921447
"type": "object",
@@ -21561,6 +21579,10 @@
2156121579
"description": "Limits define the limitations such as buffer read batch size for all the vertices of a pipeline, they could be overridden by each vertex's settings",
2156221580
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PipelineLimits"
2156321581
},
21582+
"ordered": {
21583+
"description": "Ordered enables order-preserving processing for the entire pipeline. When enabled, messages will be processed in their arrival order (FIFO within each partition). This can be overridden at the vertex level.",
21584+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered"
21585+
},
2156421586
"sideInputs": {
2156521587
"description": "SideInputs defines the Side Inputs of a pipeline.",
2156621588
"type": "array",
@@ -22976,6 +22998,10 @@
2297622998
"type": "string"
2297722999
}
2297823000
},
23001+
"ordered": {
23002+
"description": "Ordered enables ordered processing for this vertex, overrides pipeline-level setting. When enabled, messages will be processed in order based on their event time. Note: Reduce vertices ignore this setting as they are already partitioned.",
23003+
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Ordered"
23004+
},
2297923005
"partitions": {
2298023006
"description": "Number of partitions of the vertex owned buffers. It applies to udf and sink vertices only.",
2298123007
"type": "integer",

‎config/advanced-install/namespaced-numaflow-server.yaml‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -146,19 +146,20 @@ data:
146146
url: http://prometheus-operated.monitoring.svc.cluster.local:9090\npatterns:\n-
147147
name: vertex_gauge\n objects: \n - vertex\n title: Vertex Gauge Metrics\n
148148
\ description: This pattern represents the gauge metrics for a vertex across different
149-
dimensions\n expr: |\n sum($metric_name{$filters}) by ($dimension, period)\n
150-
\ params:\n - name: start_time\n required: false\n - name: end_time\n
151-
\ required: false\n metrics:\n - metric_name: vertex_pending_messages\n
152-
\ display_name: Vertex Pending Messages\n metric_description: This gauge
153-
metric keeps track of the total number of messages that are waiting to be processed
154-
over varying time frames of 1min, 5min, 15min and default period of 2min. \n required_filters:\n
149+
dimensions\n # below expression calulates max pending messages(across replicas)
150+
for each partition and then sums up pending of all partitions.\n # This is done
151+
since in versions < 1.6, we used to emit pending metric for each pod.\n # With
152+
version >= 1.6, we emit pending metric from daemon\n expr: |\n sum(max($metric_name{$filters})
153+
by ($dimension,partition_name,period)) by ($dimension, period)\n params:\n -
154+
name: start_time\n required: false\n - name: end_time\n required:
155+
false\n metrics:\n - metric_name: vertex_pending_messages\n display_name:
156+
Vertex Pending Messages\n metric_description: This gauge metric keeps track
157+
of the total number of messages that are waiting to be processed over varying
158+
time frames of 1min, 5min, 15min and default period of 2min. \n required_filters:\n
155159
\ - namespace\n - pipeline\n - vertex\n dimensions:\n
156-
\ - name: pod\n # expr: optional expression for prometheus query\n
160+
\ - name: vertex\n # expr: optional expression for prometheus query\n
157161
\ # overrides the default expression\n filters:\n -
158-
name: pod\n required: false\n - name: period\n required:
159-
false\n - name: vertex\n # expr: optional expression for prometheus
160-
query\n # overrides the default expression\n filters:\n -
161-
name: period\n required: false\n\n- name: mono_vertex_gauge\n objects:
162+
name: period\n required: false\n- name: mono_vertex_gauge\n objects:
162163
\n - mono-vertex\n title: MonoVertex Gauge Metrics\n description: This pattern
163164
represents the gauge metrics for a mono-vertex across different dimensions\n expr:
164165
|\n sum($metric_name{$filters}) by ($dimension, period)\n params:\n - name:

‎config/advanced-install/numaflow-server.yaml‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -153,19 +153,20 @@ data:
153153
url: http://prometheus-operated.monitoring.svc.cluster.local:9090\npatterns:\n-
154154
name: vertex_gauge\n objects: \n - vertex\n title: Vertex Gauge Metrics\n
155155
\ description: This pattern represents the gauge metrics for a vertex across different
156-
dimensions\n expr: |\n sum($metric_name{$filters}) by ($dimension, period)\n
157-
\ params:\n - name: start_time\n required: false\n - name: end_time\n
158-
\ required: false\n metrics:\n - metric_name: vertex_pending_messages\n
159-
\ display_name: Vertex Pending Messages\n metric_description: This gauge
160-
metric keeps track of the total number of messages that are waiting to be processed
161-
over varying time frames of 1min, 5min, 15min and default period of 2min. \n required_filters:\n
156+
dimensions\n # below expression calulates max pending messages(across replicas)
157+
for each partition and then sums up pending of all partitions.\n # This is done
158+
since in versions < 1.6, we used to emit pending metric for each pod.\n # With
159+
version >= 1.6, we emit pending metric from daemon\n expr: |\n sum(max($metric_name{$filters})
160+
by ($dimension,partition_name,period)) by ($dimension, period)\n params:\n -
161+
name: start_time\n required: false\n - name: end_time\n required:
162+
false\n metrics:\n - metric_name: vertex_pending_messages\n display_name:
163+
Vertex Pending Messages\n metric_description: This gauge metric keeps track
164+
of the total number of messages that are waiting to be processed over varying
165+
time frames of 1min, 5min, 15min and default period of 2min. \n required_filters:\n
162166
\ - namespace\n - pipeline\n - vertex\n dimensions:\n
163-
\ - name: pod\n # expr: optional expression for prometheus query\n
167+
\ - name: vertex\n # expr: optional expression for prometheus query\n
164168
\ # overrides the default expression\n filters:\n -
165-
name: pod\n required: false\n - name: period\n required:
166-
false\n - name: vertex\n # expr: optional expression for prometheus
167-
query\n # overrides the default expression\n filters:\n -
168-
name: period\n required: false\n\n- name: mono_vertex_gauge\n objects:
169+
name: period\n required: false\n- name: mono_vertex_gauge\n objects:
169170
\n - mono-vertex\n title: MonoVertex Gauge Metrics\n description: This pattern
170171
represents the gauge metrics for a mono-vertex across different dimensions\n expr:
171172
|\n sum($metric_name{$filters}) by ($dimension, period)\n params:\n - name:

0 commit comments

Comments
 (0)