diff --git a/changelog/unreleased/issue-26142.toml b/changelog/unreleased/issue-26142.toml new file mode 100644 index 000000000000..5af0c241c793 --- /dev/null +++ b/changelog/unreleased/issue-26142.toml @@ -0,0 +1,5 @@ +type = "a" +message = "Add `date_diff` pipeline function to compute the difference between two date objects." + +issues = ["26142"] +pulls = ["26143"] diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index b3fbc2a5b4dc..4e884accd564 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -41,6 +41,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.DateDiff; import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate; @@ -246,6 +247,7 @@ protected void configure() { addMessageProcessorFunction(ParseUnixMilliseconds.NAME, ParseUnixMilliseconds.class); addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class); addMessageProcessorFunction(FormatDate.NAME, FormatDate.class); + addMessageProcessorFunction(DateDiff.NAME, DateDiff.class); addMessageProcessorFunction(Years.NAME, Years.class); addMessageProcessorFunction(Months.NAME, Months.class); addMessageProcessorFunction(Weeks.NAME, Weeks.class); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateDiff.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateDiff.java new file mode 100644 index 000000000000..4bf0e473c35a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateDiff.java @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.rulebuilder.RuleBuilderFunctionGroup; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +import java.util.Map; + +public class DateDiff extends AbstractFunction> { + + public static final String NAME = "date_diff"; + + private static final String LEFT = "left"; + private static final String RIGHT = "right"; + private static final String ABSOLUTE = "absolute"; + + private static final long MS_PER_SECOND = 1000L; + private static final long MS_PER_MINUTE = 60L * MS_PER_SECOND; + private static final long MS_PER_HOUR = 60L * MS_PER_MINUTE; + private static final long MS_PER_DAY = 24L * MS_PER_HOUR; + private static final long MS_PER_WEEK = 7L * MS_PER_DAY; + + private final ParameterDescriptor left; + private final ParameterDescriptor right; + private final ParameterDescriptor absolute; + + public DateDiff() { + left = ParameterDescriptor.type(LEFT, DateTime.class) + .description("Start of the interval. May be before or after the end; the result is signed by default (end - start).") + .ruleBuilderVariable() + .build(); + right = ParameterDescriptor.type(RIGHT, DateTime.class) + .description("End of the interval. May be before or after the start.") + .build(); + absolute = ParameterDescriptor.bool(ABSOLUTE) + .optional() + .description("If true, return absolute values; otherwise the result is signed (end - start). Defaults to false.") + .build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final DateTime leftValue = left.required(args, context); + final DateTime rightValue = right.required(args, context); + if (leftValue == null || rightValue == null) { + return null; + } + final boolean abs = absolute.optional(args, context).orElse(false); + + final long signedMillis = new Duration(leftValue, rightValue).getMillis(); + final long value = (abs && signedMillis < 0) ? -signedMillis : signedMillis; + + return ImmutableMap.builder() + .put("millis", value) + .put("seconds", roundDiv(value, MS_PER_SECOND)) + .put("minutes", roundDiv(value, MS_PER_MINUTE)) + .put("hours", roundDiv(value, MS_PER_HOUR)) + .put("days", roundDiv(value, MS_PER_DAY)) + .put("weeks", roundDiv(value, MS_PER_WEEK)) + .put("direction", direction(signedMillis)) + .put("friendly", friendly(value)) + .build(); + } + + /** + * Divide {@code value} by {@code divisor} with half-away-from-zero rounding, symmetric + * across positive and negative values. e.g. 2350000ms ÷ 60000 = 39.17 → 39 minutes; + * 2370000ms ÷ 60000 = 39.5 → 40 minutes; -2370000ms → -40 minutes. + */ + private static long roundDiv(long value, long divisor) { + final long half = divisor / 2; + return value >= 0 ? (value + half) / divisor : (value - half) / divisor; + } + + /** + * Describes {@code right} relative to {@code left}. Computed from the signed millis, + * so direction is preserved even when {@code absolute=true} strips the sign from the + * numeric components. + */ + private static String direction(long signedMillis) { + if (signedMillis > 0) { + return "ahead"; + } + if (signedMillis < 0) { + return "behind"; + } + return "equal"; + } + + /** + * Human-readable rendering of the (possibly signed) interval. Zero-valued components are + * omitted. Sub-second remainder is included as a "ms" component only when the total + * interval is below one minute, so long intervals aren't cluttered with millisecond noise; + * the raw {@code millis} field always carries the exact value. + */ + private static String friendly(long signedMillis) { + if (signedMillis == 0) { + return "0 ms"; + } + final boolean neg = signedMillis < 0; + final long m = neg ? -signedMillis : signedMillis; + final StringBuilder sb = new StringBuilder(); + if (neg) { + sb.append('-'); + } + final long weeks = m / MS_PER_WEEK; + final long days = (m / MS_PER_DAY) % 7; + final long hours = (m / MS_PER_HOUR) % 24; + final long minutes = (m / MS_PER_MINUTE) % 60; + final long seconds = (m / MS_PER_SECOND) % 60; + final long millis = m % MS_PER_SECOND; + appendPart(sb, weeks, "week", "weeks"); + appendPart(sb, days, "day", "days"); + appendPart(sb, hours, "hour", "hours"); + appendPart(sb, minutes, "minute", "minutes"); + appendPart(sb, seconds, "second", "seconds"); + // Include sub-second remainder when the interval is below a minute, so callers see + // precision for short deltas without "2 weeks ... 47 ms" noise on long ones. + if (millis > 0 && m < MS_PER_MINUTE) { + appendPart(sb, millis, "ms", "ms"); + } + return sb.toString(); + } + + private static void appendPart(StringBuilder sb, long value, String singular, String plural) { + if (value == 0) { + return; + } + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '-') { + sb.append(' '); + } + sb.append(value).append(' ').append(value == 1 ? singular : plural); + } + + @Override + public FunctionDescriptor> descriptor() { + @SuppressWarnings({"unchecked", "rawtypes"}) + final Class> returnType = (Class) Map.class; + return FunctionDescriptor.>builder() + .name(NAME) + .returnType(returnType) + .params(ImmutableList.of(left, right, absolute)) + .description("Returns the difference between two dates as a map. The numeric units " + + "(millis, seconds, minutes, hours, days, weeks) are rounded to the nearest whole " + + "unit. The map also contains 'direction', which describes the end relative to the " + + "start as \"ahead\", \"behind\", or \"equal\", and 'friendly', a human-readable " + + "breakdown of the interval. Numeric values are signed by default (end - start). " + + "Pass absolute=true to return absolute values; direction is always derived from " + + "the signed result and is preserved.") + .ruleBuilderEnabled() + .ruleBuilderName("Date difference") + .ruleBuilderTitle("Difference between '${left}' and '${right}'") + .ruleBuilderFunctionGroup(RuleBuilderFunctionGroup.DATE) + .build(); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 110b70952428..4800245ad8ed 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -51,6 +51,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.DateDiff; import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate; @@ -321,6 +322,7 @@ public static void registerFunctions() { functions.put(ParseDate.NAME, new ParseDate()); functions.put(ParseUnixMilliseconds.NAME, new ParseUnixMilliseconds()); functions.put(FormatDate.NAME, new FormatDate()); + functions.put(DateDiff.NAME, new DateDiff()); functions.put(Years.NAME, new Years()); functions.put(Months.NAME, new Months()); @@ -1338,6 +1340,140 @@ void dateArithmetic() { } } + @Test + void dateDiff() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + try { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + + // 2-day positive interval covers every numeric unit + direction + friendly + assertThat(message.getField("pos_millis")).isEqualTo(172_800_000L); + assertThat(message.getField("pos_seconds")).isEqualTo(172_800L); + assertThat(message.getField("pos_minutes")).isEqualTo(2_880L); + assertThat(message.getField("pos_hours")).isEqualTo(48L); + assertThat(message.getField("pos_days")).isEqualTo(2L); + assertThat(message.getField("pos_weeks")).isEqualTo(0L); + assertThat(message.getField("pos_direction")).isEqualTo("ahead"); + assertThat(message.getField("pos_friendly")).isEqualTo("2 days"); + + // Swapping args gives a signed result + "behind" direction + assertThat(message.getField("neg_millis")).isEqualTo(-172_800_000L); + assertThat(message.getField("neg_direction")).isEqualTo("behind"); + + // absolute=true strips sign from numeric values but preserves direction + assertThat(message.getField("abs_millis")).isEqualTo(172_800_000L); + assertThat(message.getField("abs_direction")).isEqualTo("behind"); + + // Equal instants + assertThat(message.getField("eq_direction")).isEqualTo("equal"); + assertThat(message.getField("eq_friendly")).isEqualTo("0 ms"); + + // Friendly behaviors: multi-component, sub-second remainder, suppression at ≥ 1 minute + assertThat(message.getField("mixed_friendly")).isEqualTo("1 week 1 day 3 hours 15 minutes"); + assertThat(message.getField("sub_friendly")).isEqualTo("1 second 500 ms"); + assertThat(message.getField("over_minute_friendly")).isEqualTo("1 minute"); + + // Half-away-from-zero rounding (1m30s sits exactly on the boundary → 2 minutes) + assertThat(message.getField("rnd_minutes")).isEqualTo(2L); + + // Realistic flow via to_date($message.timestamp); clock pins it at GRAYLOG_EPOCH + assertThat(message.getField("session_minutes")).isEqualTo(30L); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } + + @Test + void dateDiffPrExamples() { + final InstantMillisProvider clock = new InstantMillisProvider(DateTime.parse("2025-05-27T14:00:00.000Z")); + DateTimeUtils.setCurrentMillisProvider(clock); + try { + // Example 1: VPN session duration + final String vpnRule = + "rule \"vpn session duration\"\n" + + "when\n" + + " has_field(\"acct_session_start\")\n" + + "then\n" + + " let start_dt = parse_date(value: to_string($message.acct_session_start),\n" + + " pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" + + " let end_dt = to_date($message.timestamp);\n" + + "\n" + + " let session = date_diff(start_dt, end_dt);\n" + + " set_field(\"session_seconds\", session.seconds);\n" + + " set_field(\"session_minutes\", session.minutes);\n" + + " set_field(\"session_hours\", session.hours);\n" + + "end"; + final Rule vpn = parser.parseRule(vpnRule, true); + final Message vpnMsg = evaluateRule(vpn, msg -> msg.addField("acct_session_start", "2025-05-27T13:42:10.000+0000")); + assertThat(vpnMsg).isNotNull(); + // 17m 50s elapsed = 1070s; minutes rounds to 18 (half-away-from-zero), hours rounds to 0 + assertThat(vpnMsg.getField("session_seconds")).isEqualTo(1070L); + assertThat(vpnMsg.getField("session_minutes")).isEqualTo(18L); + assertThat(vpnMsg.getField("session_hours")).isEqualTo(0L); + + // Example 2: Account age at login + final String ageRule = + "rule \"tag new account logins\"\n" + + "when\n" + + " has_field(\"event_type\") && to_string($message.event_type) == \"user_login\"\n" + + "then\n" + + " let created = parse_date(value: to_string($message.user_created),\n" + + " pattern: \"MM/dd/yyyy\");\n" + + " let age = date_diff(left: created, right: now(), absolute: true);\n" + + "\n" + + " set_field(\"account_age_days\", age.days);\n" + + " set_field(\"account_is_new\", to_long(age.days) < 7);\n" + + "end"; + final Rule ageR = parser.parseRule(ageRule, true); + final Message ageMsgFresh = evaluateRule(ageR, msg -> { + msg.addField("event_type", "user_login"); + // 05/25/2025 parses to midnight UTC; now is 2025-05-27T14:00Z = 62h elapsed, + // which rounds to 3 days (half-away-from-zero). + msg.addField("user_created", "05/25/2025"); + }); + assertThat(ageMsgFresh).isNotNull(); + assertThat(ageMsgFresh.getField("account_age_days")).isEqualTo(3L); + assertThat(ageMsgFresh.getField("account_is_new")).isEqualTo(true); + + final Message ageMsgOld = evaluateRule(ageR, msg -> { + msg.addField("event_type", "user_login"); + msg.addField("user_created", "03/15/2024"); + }); + assertThat(ageMsgOld).isNotNull(); + assertThat(ageMsgOld.getField("account_is_new")).isEqualTo(false); + + // Example 3: HTTP request latency + final String latencyRule = + "rule \"http latency\"\n" + + "when\n" + + " has_field(\"request_received_at\") && has_field(\"response_sent_at\")\n" + + "then\n" + + " let req = parse_date(value: to_string($message.request_received_at),\n" + + " pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" + + " let res = parse_date(value: to_string($message.response_sent_at),\n" + + " pattern: \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n" + + "\n" + + " let latency = date_diff(req, res);\n" + + " set_field(\"latency_ms\", latency.millis);\n" + + " set_field(\"latency_seconds\", latency.seconds);\n" + + "end"; + final Rule lat = parser.parseRule(latencyRule, true); + final Message latMsg = evaluateRule(lat, msg -> { + msg.addField("request_received_at", "2025-05-27T13:59:59.750+0000"); + msg.addField("response_sent_at", "2025-05-27T14:00:00.123+0000"); + }); + assertThat(latMsg).isNotNull(); + assertThat(latMsg.getField("latency_ms")).isEqualTo(373L); + assertThat(latMsg.getField("latency_seconds")).isEqualTo(0L); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } + @Test void routeToStream() { final Rule rule = parser.parseRule(ruleForTest(), true); diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateDiff.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateDiff.txt new file mode 100644 index 000000000000..7bb1a7ddc9e3 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateDiff.txt @@ -0,0 +1,52 @@ +rule "date diff" +when + true +then + let earlier = parse_date(value: "2023-05-10T00:00:00.000+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + let later = parse_date(value: "2023-05-12T00:00:00.000+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + // Positive 2-day interval exercises every numeric unit, direction, and friendly + let positive = date_diff(earlier, later); + set_field("pos_millis", positive.millis); + set_field("pos_seconds", positive.seconds); + set_field("pos_minutes", positive.minutes); + set_field("pos_hours", positive.hours); + set_field("pos_days", positive.days); + set_field("pos_weeks", positive.weeks); + set_field("pos_direction", positive.direction); + set_field("pos_friendly", positive.friendly); + + // Swapped args → signed result with "behind" direction + let negative = date_diff(later, earlier); + set_field("neg_millis", negative.millis); + set_field("neg_direction", negative.direction); + + // absolute=true strips sign from numeric but preserves direction + let abs = date_diff(left: later, right: earlier, absolute: true); + set_field("abs_millis", abs.millis); + set_field("abs_direction", abs.direction); + + // Equal instants + let equal = date_diff(earlier, earlier); + set_field("eq_direction", equal.direction); + set_field("eq_friendly", equal.friendly); + + // Friendly: multi-component breakdown + let mixed_end = parse_date(value: "2023-05-18T03:15:00.000+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + set_field("mixed_friendly", date_diff(earlier, mixed_end).friendly); + + // Friendly: sub-second remainder shown for short intervals, suppressed for ≥ 1 minute + let one_and_a_half = parse_date(value: "2023-05-10T00:00:01.500+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + let one_min_plus = parse_date(value: "2023-05-10T00:01:00.500+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + set_field("sub_friendly", date_diff(earlier, one_and_a_half).friendly); + set_field("over_minute_friendly", date_diff(earlier, one_min_plus).friendly); + + // Half-away-from-zero rounding on numeric keys (1m30s sits exactly on the boundary) + let m30 = parse_date(value: "2023-05-10T00:01:30.000+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + set_field("rnd_minutes", date_diff(earlier, m30).minutes); + + // Realistic flow: to_date($message.timestamp) against a parsed field. + // Test fixture's message timestamp is GRAYLOG_EPOCH = 2010-07-30T16:03:25Z. + let session_open = parse_date(value: "2010-07-30T15:33:25.000+0000", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + set_field("session_minutes", date_diff(session_open, to_date($message.timestamp)).minutes); +end