Skip to content

Commit edb2252

Browse files
docs: Add SDK examples to Data Transformers Re: numaproj#3041 (numaproj#3047)
Signed-off-by: Rachel McGuigan <saturnsmoon64@gmail.com> Signed-off-by: Vigith Maurice <vigith@gmail.com> Co-authored-by: Vigith Maurice <vigith@gmail.com>
1 parent 6e6a3fc commit edb2252

1 file changed

Lines changed: 131 additions & 3 deletions

File tree

docs/user-guide/sources/transformer/overview.md

Lines changed: 131 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,137 @@ func main() {
6666

6767
Check the links below to see another transformer example in various programming languages, where we apply conditional forwarding based on the input event time.
6868

69-
- [Python](https://github.com/numaproj/numaflow-python/tree/main/packages/pynumaflow/examples/sourcetransform/event_time_filter)
70-
- [Golang](https://github.com/numaproj/numaflow-go/tree/main/examples/sourcetransformer/event_time_filter)
71-
- [Java](https://github.com/numaproj/numaflow-java/tree/main/examples/src/main/java/io/numaproj/numaflow/examples/sourcetransformer/eventtimefilter)
69+
=== "Python"
70+
71+
```python
72+
def my_handler(keys: list[str], datum: Datum) -> Messages:
73+
val = datum.value
74+
event_time = datum.event_time
75+
messages = Messages()
76+
77+
if event_time < january_first_2022:
78+
logging.info("Got event time:%s, it is before 2022, so dropping", event_time)
79+
messages.append(Message.to_drop(event_time))
80+
elif event_time < january_first_2023:
81+
logging.info(
82+
"Got event time:%s, it is within year 2022, so forwarding to within_year_2022",
83+
event_time,
84+
)
85+
messages.append(
86+
Message(value=val, event_time=january_first_2022, tags=["within_year_2022"])
87+
)
88+
else:
89+
logging.info(
90+
"Got event time:%s, it is after year 2022, so forwarding to after_year_2022", event_time
91+
)
92+
messages.append(Message(value=val, event_time=january_first_2023, tags=["after_year_2022"]))
93+
94+
return messages
95+
96+
```
97+
- [Python full example on numaflow-python on Github](https://github.com/numaproj/numaflow-python/blob/main/packages/pynumaflow/examples/sourcetransform/event_time_filter/example.py)
98+
99+
=== "Go"
100+
101+
```go
102+
package main
103+
104+
import (
105+
"context"
106+
"log"
107+
108+
"event_time_filter/impl"
109+
110+
"github.com/numaproj/numaflow-go/pkg/sourcetransformer"
111+
)
112+
113+
func transform(_ context.Context, keys []string, d sourcetransformer.Datum) sourcetransformer.Messages {
114+
return impl.FilterEventTime(keys, d)
115+
}
116+
117+
func main() {
118+
err := sourcetransformer.NewServer(sourcetransformer.SourceTransformFunc(transform)).Start(context.Background())
119+
if err != nil {
120+
log.Panic("Failed to start source transform server: ", err)
121+
}
122+
}
123+
```
124+
- [Golang full example on numaflow-go Github](https://github.com/numaproj/numaflow-go/blob/main/examples/sourcetransformer/event_time_filter/main.go)
125+
126+
=== "Rust"
127+
128+
```rust
129+
fn transformer_fn(_ctx: &(), keys: &[String], datum: Datum) -> MessagesBuilder {
130+
let payload = datum.value();
131+
let json: Value = serde_json::from_slice(payload).unwrap_or_default();
132+
133+
// Get event time override (optional)
134+
let mut event_time = datum.event_time();
135+
if let Some(ts) = json.get("ts").and_then(|v| v.as_i64()) {
136+
event_time = std::time::UNIX_EPOCH + std::time::Duration::from_secs(ts as u64);
137+
}
138+
139+
// Filter out messages
140+
let should_drop = json.get("drop").and_then(|v| v.as_bool()).unwrap_or(false);
141+
if should_drop {
142+
MessagesBuilder::default().append(MessageT::to_drop(event_time))
143+
} else {
144+
MessagesBuilder::default().append(
145+
MessageT::new(payload.to_vec(), event_time).with_keys(keys.to_vec()),
146+
)
147+
}
148+
}
149+
```
150+
- [Rust full example on numaflow-rs Github](https://github.com/numaproj/numaflow-rs/blob/main/numaflow/proto/sourcetransform.proto)
151+
152+
153+
=== "Java"
154+
155+
```java
156+
public class EventTimeFilterFunction extends SourceTransformer {
157+
158+
private static final Instant januaryFirst2022 = Instant.ofEpochMilli(1640995200000L);
159+
private static final Instant januaryFirst2023 = Instant.ofEpochMilli(1672531200000L);
160+
161+
public static void main(String[] args) throws Exception {
162+
Server server = new Server(new EventTimeFilterFunction());
163+
164+
// Start the server
165+
server.start();
166+
167+
// wait for the server to shut down
168+
server.awaitTermination();
169+
}
170+
171+
public MessageList processMessage(String[] keys, Datum data) {
172+
Instant eventTime = data.getEventTime();
173+
174+
if (eventTime.isBefore(januaryFirst2022)) {
175+
return MessageList.newBuilder().addMessage(Message.toDrop(eventTime)).build();
176+
} else if (eventTime.isBefore(januaryFirst2023)) {
177+
return MessageList
178+
.newBuilder()
179+
.addMessage(
180+
new Message(
181+
data.getValue(),
182+
januaryFirst2022,
183+
null,
184+
new String[]{"within_year_2022"}))
185+
.build();
186+
} else {
187+
return MessageList
188+
.newBuilder()
189+
.addMessage(new Message(
190+
data.getValue(),
191+
januaryFirst2023,
192+
null,
193+
new String[]{"after_year_2022"}))
194+
.build();
195+
}
196+
}
197+
}
198+
```
199+
- [Java full example of numaflow-java on Github](https://github.com/numaproj/numaflow-java/blob/main/examples/src/main/java/io/numaproj/numaflow/examples/sourcetransformer/eventtimefilter/EventTimeFilterFunction.java)
72200

73201
After building a docker image for the written transformer, specify the image as below in the source vertex spec.
74202

0 commit comments

Comments
 (0)