Skip to content

Commit 08b1cab

Browse files
danotorreydanotorrey
authored
Add date_diff pipeline function for date difference calculations (#26143)
1 parent c244f56 commit 08b1cab

5 files changed

Lines changed: 374 additions & 0 deletions

File tree

‎changelog/unreleased/issue-26142.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "a"
2+
message = "Add `date_diff` pipeline function to compute the difference between two date objects."
3+
4+
issues = ["26142"]
5+
pulls = ["26143"]

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion;
4242
import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion;
4343
import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion;
44+
import org.graylog.plugins.pipelineprocessor.functions.dates.DateDiff;
4445
import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate;
4546
import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate;
4647
import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate;
@@ -246,6 +247,7 @@ protected void configure() {
246247
addMessageProcessorFunction(ParseUnixMilliseconds.NAME, ParseUnixMilliseconds.class);
247248
addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class);
248249
addMessageProcessorFunction(FormatDate.NAME, FormatDate.class);
250+
addMessageProcessorFunction(DateDiff.NAME, DateDiff.class);
249251
addMessageProcessorFunction(Years.NAME, Years.class);
250252
addMessageProcessorFunction(Months.NAME, Months.class);
251253
addMessageProcessorFunction(Weeks.NAME, Weeks.class);

‎graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateDiff.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
package org.graylog.plugins.pipelineprocessor.functions.dates;
18+
19+
import com.google.common.collect.ImmutableList;
20+
import com.google.common.collect.ImmutableMap;
21+
import org.graylog.plugins.pipelineprocessor.EvaluationContext;
22+
import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction;
23+
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
24+
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
25+
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
26+
import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup;
27+
import org.joda.time.DateTime;
28+
import org.joda.time.Duration;
29+
30+
import java.util.Map;
31+
32+
public class DateDiff extends AbstractFunction<Map<String, Object>> {
33+
34+
public static final String NAME = "date_diff";
35+
36+
private static final String LEFT = "left";
37+
private static final String RIGHT = "right";
38+
private static final String ABSOLUTE = "absolute";
39+
40+
private static final long MS_PER_SECOND = 1000L;
41+
private static final long MS_PER_MINUTE = 60L * MS_PER_SECOND;
42+
private static final long MS_PER_HOUR = 60L * MS_PER_MINUTE;
43+
private static final long MS_PER_DAY = 24L * MS_PER_HOUR;
44+
private static final long MS_PER_WEEK = 7L * MS_PER_DAY;
45+
46+
private final ParameterDescriptor<DateTime, DateTime> left;
47+
private final ParameterDescriptor<DateTime, DateTime> right;
48+
private final ParameterDescriptor<Boolean, Boolean> absolute;
49+
50+
public DateDiff() {
51+
left = ParameterDescriptor.type(LEFT, DateTime.class)
52+
.description("Start of the interval. May be before or after the end; the result is signed by default (end - start).")
53+
.ruleBuilderVariable()
54+
.build();
55+
right = ParameterDescriptor.type(RIGHT, DateTime.class)
56+
.description("End of the interval. May be before or after the start.")
57+
.build();
58+
absolute = ParameterDescriptor.bool(ABSOLUTE)
59+
.optional()
60+
.description("If true, return absolute values; otherwise the result is signed (end - start). Defaults to false.")
61+
.build();
62+
}
63+
64+
@Override
65+
public Map<String, Object> evaluate(FunctionArgs args, EvaluationContext context) {
66+
final DateTime leftValue = left.required(args, context);
67+
final DateTime rightValue = right.required(args, context);
68+
if (leftValue == null || rightValue == null) {
69+
return null;
70+
}
71+
final boolean abs = absolute.optional(args, context).orElse(false);
72+
73+
final long signedMillis = new Duration(leftValue, rightValue).getMillis();
74+
final long value = (abs && signedMillis < 0) ? -signedMillis : signedMillis;
75+
76+
return ImmutableMap.<String, Object>builder()
77+
.put("millis", value)
78+
.put("seconds", roundDiv(value, MS_PER_SECOND))
79+
.put("minutes", roundDiv(value, MS_PER_MINUTE))
80+
.put("hours", roundDiv(value, MS_PER_HOUR))
81+
.put("days", roundDiv(value, MS_PER_DAY))
82+
.put("weeks", roundDiv(value, MS_PER_WEEK))
83+
.put("direction", direction(signedMillis))
84+
.put("friendly", friendly(value))
85+
.build();
86+
}
87+
88+
/**
89+
* Divide {@code value} by {@code divisor} with half-away-from-zero rounding, symmetric
90+
* across positive and negative values. e.g. 2350000ms ÷ 60000 = 39.17 → 39 minutes;
91+
* 2370000ms ÷ 60000 = 39.5 → 40 minutes; -2370000ms → -40 minutes.
92+
*/
93+
private static long roundDiv(long value, long divisor) {
94+
final long half = divisor / 2;
95+
return value >= 0 ? (value + half) / divisor : (value - half) / divisor;
96+
}
97+
98+
/**
99+
* Describes {@code right} relative to {@code left}. Computed from the signed millis,
100+
* so direction is preserved even when {@code absolute=true} strips the sign from the
101+
* numeric components.
102+
*/
103+
private static String direction(long signedMillis) {
104+
if (signedMillis > 0) {
105+
return "ahead";
106+
}
107+
if (signedMillis < 0) {
108+
return "behind";
109+
}
110+
return "equal";
111+
}
112+
113+
/**
114+
* Human-readable rendering of the (possibly signed) interval. Zero-valued components are
115+
* omitted. Sub-second remainder is included as a "ms" component only when the total
116+
* interval is below one minute, so long intervals aren't cluttered with millisecond noise;
117+
* the raw {@code millis} field always carries the exact value.
118+
*/
119+
private static String friendly(long signedMillis) {
120+
if (signedMillis == 0) {
121+
return "0 ms";
122+
}
123+
final boolean neg = signedMillis < 0;
124+
final long m = neg ? -signedMillis : signedMillis;
125+
final StringBuilder sb = new StringBuilder();
126+
if (neg) {
127+
sb.append('-');
128+
}
129+
final long weeks = m / MS_PER_WEEK;
130+
final long days = (m / MS_PER_DAY) % 7;
131+
final long hours = (m / MS_PER_HOUR) % 24;
132+
final long minutes = (m / MS_PER_MINUTE) % 60;
133+
final long seconds = (m / MS_PER_SECOND) % 60;
134+
final long millis = m % MS_PER_SECOND;
135+
appendPart(sb, weeks, "week", "weeks");
136+
appendPart(sb, days, "day", "days");
137+
appendPart(sb, hours, "hour", "hours");
138+
appendPart(sb, minutes, "minute", "minutes");
139+
appendPart(sb, seconds, "second", "seconds");
140+
// Include sub-second remainder when the interval is below a minute, so callers see
141+
// precision for short deltas without "2 weeks ... 47 ms" noise on long ones.
142+
if (millis > 0 && m < MS_PER_MINUTE) {
143+
appendPart(sb, millis, "ms", "ms");
144+
}
145+
return sb.toString();
146+
}
147+
148+
private static void appendPart(StringBuilder sb, long value, String singular, String plural) {
149+
if (value == 0) {
150+
return;
151+
}
152+
if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '-') {
153+
sb.append(' ');
154+
}
155+
sb.append(value).append(' ').append(value == 1 ? singular : plural);
156+
}
157+
158+
@Override
159+
public FunctionDescriptor<Map<String, Object>> descriptor() {
160+
@SuppressWarnings({"unchecked", "rawtypes"})
161+
final Class<? extends Map<String, Object>> returnType = (Class) Map.class;
162+
return FunctionDescriptor.<Map<String, Object>>builder()
163+
.name(NAME)
164+
.returnType(returnType)
165+
.params(ImmutableList.of(left, right, absolute))
166+
.description("Returns the difference between two dates as a map. The numeric units " +
167+
"(millis, seconds, minutes, hours, days, weeks) are rounded to the nearest whole " +
168+
"unit. The map also contains 'direction', which describes the end relative to the " +
169+
"start as \"ahead\", \"behind\", or \"equal\", and 'friendly', a human-readable " +
170+
"breakdown of the interval. Numeric values are signed by default (end - start). " +
171+
"Pass absolute=true to return absolute values; direction is always derived from " +
172+
"the signed result and is preserved.")
173+
.ruleBuilderEnabled()
174+
.ruleBuilderName("Date difference")
175+
.ruleBuilderTitle("Difference between '${left}' and '${right}'")
176+
.ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.DATE)
177+
.build();
178+
}
179+
}

‎graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion;
5252
import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion;
5353
import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion;
54+
import org.graylog.plugins.pipelineprocessor.functions.dates.DateDiff;
5455
import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate;
5556
import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate;
5657
import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate;
@@ -321,6 +322,7 @@ public static void registerFunctions() {
321322
functions.put(ParseDate.NAME, new ParseDate());
322323
functions.put(ParseUnixMilliseconds.NAME, new ParseUnixMilliseconds());
323324
functions.put(FormatDate.NAME, new FormatDate());
325+
functions.put(DateDiff.NAME, new DateDiff());
324326

325327
functions.put(Years.NAME, new Years());
326328
functions.put(Months.NAME, new Months());
@@ -1338,6 +1340,140 @@ void dateArithmetic() {
13381340
}
13391341
}
13401342

1343+
@Test
1344+
void dateDiff() {
1345+
final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH);
1346+
DateTimeUtils.setCurrentMillisProvider(clock);
1347+
try {
1348+
final Rule rule = parser.parseRule(ruleForTest(), true);
1349+
final Message message = evaluateRule(rule);
1350+
1351+
assertThat(message).isNotNull();
1352+
1353+
// 2-day positive interval covers every numeric unit + direction + friendly
1354+
assertThat(message.getField("pos_millis")).isEqualTo(172_800_000L);
1355+
assertThat(message.getField("pos_seconds")).isEqualTo(172_800L);
1356+
assertThat(message.getField("pos_minutes")).isEqualTo(2_880L);
1357+
assertThat(message.getField("pos_hours")).isEqualTo(48L);
1358+
assertThat(message.getField("pos_days")).isEqualTo(2L);
1359+
assertThat(message.getField("pos_weeks")).isEqualTo(0L);
1360+
assertThat(message.getField("pos_direction")).isEqualTo("ahead");
1361+
assertThat(message.getField("pos_friendly")).isEqualTo("2 days");
1362+
1363+
// Swapping args gives a signed result + "behind" direction
1364+
assertThat(message.getField("neg_millis")).isEqualTo(-172_800_000L);
1365+
assertThat(message.getField("neg_direction")).isEqualTo("behind");
1366+
1367+
// absolute=true strips sign from numeric values but preserves direction
1368+
assertThat(message.getField("abs_millis")).isEqualTo(172_800_000L);
1369+
assertThat(message.getField("abs_direction")).isEqualTo("behind");
1370+
1371+
// Equal instants
1372+
assertThat(message.getField("eq_direction")).isEqualTo("equal");
1373+
assertThat(message.getField("eq_friendly")).isEqualTo("0 ms");
1374+
1375+
// Friendly behaviors: multi-component, sub-second remainder, suppression at ≥ 1 minute
1376+
assertThat(message.getField("mixed_friendly")).isEqualTo("1 week 1 day 3 hours 15 minutes");
1377+
assertThat(message.getField("sub_friendly")).isEqualTo("1 second 500 ms");
1378+
assertThat(message.getField("over_minute_friendly")).isEqualTo("1 minute");
1379+
1380+
// Half-away-from-zero rounding (1m30s sits exactly on the boundary → 2 minutes)
1381+
assertThat(message.getField("rnd_minutes")).isEqualTo(2L);
1382+
1383+
// Realistic flow via to_date($message.timestamp); clock pins it at GRAYLOG_EPOCH
1384+
assertThat(message.getField("session_minutes")).isEqualTo(30L);
1385+
} finally {
1386+
DateTimeUtils.setCurrentMillisSystem();
1387+
}
1388+
}
1389+
1390+
@Test
1391+
void dateDiffPrExamples() {
1392+
final InstantMillisProvider clock = new InstantMillisProvider(DateTime.parse("2025-05-27T14:00:00.000Z"));
1393+
DateTimeUtils.setCurrentMillisProvider(clock);
1394+
try {
1395+
// Example 1: VPN session duration
1396+
final String vpnRule =
1397+
"rule \"vpn session duration\"\n" +
1398+
"when\n" +
1399+
" has_field(\"acct_session_start\")\n" +
1400+
"then\n" +
1401+
" let start_dt = parse_date(value: to_string($message.acct_session_start),\n" +
1402+
" pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" +
1403+
" let end_dt = to_date($message.timestamp);\n" +
1404+
"\n" +
1405+
" let session = date_diff(start_dt, end_dt);\n" +
1406+
" set_field(\"session_seconds\", session.seconds);\n" +
1407+
" set_field(\"session_minutes\", session.minutes);\n" +
1408+
" set_field(\"session_hours\", session.hours);\n" +
1409+
"end";
1410+
final Rule vpn = parser.parseRule(vpnRule, true);
1411+
final Message vpnMsg = evaluateRule(vpn, msg -> msg.addField("acct_session_start", "2025-05-27T13:42:10.000+0000"));
1412+
assertThat(vpnMsg).isNotNull();
1413+
// 17m 50s elapsed = 1070s; minutes rounds to 18 (half-away-from-zero), hours rounds to 0
1414+
assertThat(vpnMsg.getField("session_seconds")).isEqualTo(1070L);
1415+
assertThat(vpnMsg.getField("session_minutes")).isEqualTo(18L);
1416+
assertThat(vpnMsg.getField("session_hours")).isEqualTo(0L);
1417+
1418+
// Example 2: Account age at login
1419+
final String ageRule =
1420+
"rule \"tag new account logins\"\n" +
1421+
"when\n" +
1422+
" has_field(\"event_type\") && to_string($message.event_type) == \"user_login\"\n" +
1423+
"then\n" +
1424+
" let created = parse_date(value: to_string($message.user_created),\n" +
1425+
" pattern: \"MM/dd/yyyy\");\n" +
1426+
" let age = date_diff(left: created, right: now(), absolute: true);\n" +
1427+
"\n" +
1428+
" set_field(\"account_age_days\", age.days);\n" +
1429+
" set_field(\"account_is_new\", to_long(age.days) < 7);\n" +
1430+
"end";
1431+
final Rule ageR = parser.parseRule(ageRule, true);
1432+
final Message ageMsgFresh = evaluateRule(ageR, msg -> {
1433+
msg.addField("event_type", "user_login");
1434+
// 05/25/2025 parses to midnight UTC; now is 2025-05-27T14:00Z = 62h elapsed,
1435+
// which rounds to 3 days (half-away-from-zero).
1436+
msg.addField("user_created", "05/25/2025");
1437+
});
1438+
assertThat(ageMsgFresh).isNotNull();
1439+
assertThat(ageMsgFresh.getField("account_age_days")).isEqualTo(3L);
1440+
assertThat(ageMsgFresh.getField("account_is_new")).isEqualTo(true);
1441+
1442+
final Message ageMsgOld = evaluateRule(ageR, msg -> {
1443+
msg.addField("event_type", "user_login");
1444+
msg.addField("user_created", "03/15/2024");
1445+
});
1446+
assertThat(ageMsgOld).isNotNull();
1447+
assertThat(ageMsgOld.getField("account_is_new")).isEqualTo(false);
1448+
1449+
// Example 3: HTTP request latency
1450+
final String latencyRule =
1451+
"rule \"http latency\"\n" +
1452+
"when\n" +
1453+
" has_field(\"request_received_at\") && has_field(\"response_sent_at\")\n" +
1454+
"then\n" +
1455+
" let req = parse_date(value: to_string($message.request_received_at),\n" +
1456+
" pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" +
1457+
" let res = parse_date(value: to_string($message.response_sent_at),\n" +
1458+
" pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" +
1459+
"\n" +
1460+
" let latency = date_diff(req, res);\n" +
1461+
" set_field(\"latency_ms\", latency.millis);\n" +
1462+
" set_field(\"latency_seconds\", latency.seconds);\n" +
1463+
"end";
1464+
final Rule lat = parser.parseRule(latencyRule, true);
1465+
final Message latMsg = evaluateRule(lat, msg -> {
1466+
msg.addField("request_received_at", "2025-05-27T13:59:59.750+0000");
1467+
msg.addField("response_sent_at", "2025-05-27T14:00:00.123+0000");
1468+
});
1469+
assertThat(latMsg).isNotNull();
1470+
assertThat(latMsg.getField("latency_ms")).isEqualTo(373L);
1471+
assertThat(latMsg.getField("latency_seconds")).isEqualTo(0L);
1472+
} finally {
1473+
DateTimeUtils.setCurrentMillisSystem();
1474+
}
1475+
}
1476+
13411477
@Test
13421478
void routeToStream() {
13431479
final Rule rule = parser.parseRule(ruleForTest(), true);

0 commit comments

Comments
 (0)