Skip to content

Commit 6980ad9

Browse files
committed
Refactor timewrap: extract TimewrapUtils, add precise calendar arithmetic
- Extract all timewrap helper methods to TimewrapUtils.java in calcite/utils/ - Add precise EXTRACT-based period computation for month/quarter/year - Add cumDaysBeforeMonth with leap year CASE expression for precise quarter offset - Month/quarter/year period assignment now uses calendar arithmetic instead of approximate fixed-length conversions Signed-off-by: Jialiang Li <jialiang.li@hey.com> Signed-off-by: Kai Huang <ahkcs@amazon.com>
1 parent 57495d0 commit 6980ad9

3 files changed

Lines changed: 442 additions & 183 deletions

File tree

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

Lines changed: 95 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,8 @@
9595
import org.opensearch.sql.ast.expression.Alias;
9696
import org.opensearch.sql.ast.expression.AllFields;
9797
import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta;
98-
import org.opensearch.sql.ast.expression.And;
9998
import org.opensearch.sql.ast.expression.Argument;
10099
import org.opensearch.sql.ast.expression.Argument.ArgumentMap;
101-
import org.opensearch.sql.ast.expression.Compare;
102100
import org.opensearch.sql.ast.expression.Field;
103101
import org.opensearch.sql.ast.expression.Function;
104102
import org.opensearch.sql.ast.expression.Let;
@@ -180,6 +178,7 @@
180178
import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
181179
import org.opensearch.sql.calcite.utils.PPLHintUtils;
182180
import org.opensearch.sql.calcite.utils.PlanUtils;
181+
import org.opensearch.sql.calcite.utils.TimewrapUtils;
183182
import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
184183
import org.opensearch.sql.calcite.utils.WildcardUtils;
185184
import org.opensearch.sql.common.error.ErrorCode;
@@ -3828,19 +3827,14 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) {
38283827
return relBuilder.peek();
38293828
}
38303829

3831-
private static final int TIMEWRAP_MAX_PERIODS = 20;
3832-
38333830
@Override
38343831
public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) {
38353832
visitChildren(node, context);
38363833

38373834
// Signal the execution engine to strip all-null columns and rename with absolute offsets
38383835
CalcitePlanContext.stripNullColumns.set(true);
3839-
// Both align=now and align=end use _before suffix (matching Splunk behavior).
3840-
// align=end would use search end time as reference, but PPL has no search time range
3841-
// context, so both modes currently use query execution time.
38423836
CalcitePlanContext.timewrapUnitName.set(
3843-
timewrapUnitBaseName(node.getUnit(), node.getValue()) + "|_before");
3837+
TimewrapUtils.unitBaseName(node.getUnit(), node.getValue()) + "|_before");
38443838

38453839
RelBuilder b = context.relBuilder;
38463840
RexBuilder rx = context.rexBuilder;
@@ -3850,62 +3844,100 @@ public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) {
38503844
String tsFieldName = fieldNames.get(0);
38513845
List<String> valueFieldNames = fieldNames.subList(1, fieldNames.size());
38523846

3853-
long spanSec = timewrapSpanToSeconds(node.getUnit(), node.getValue());
3847+
boolean variableLength = TimewrapUtils.isVariableLengthUnit(node.getUnit());
38543848
RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT);
38553849

3856-
// Step 1: Convert timestamps to epoch seconds via UNIX_TIMESTAMP, add MAX OVER()
3857-
RexNode tsEpochExpr =
3858-
rx.makeCast(
3859-
bigintType,
3860-
rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, b.field(tsFieldName)),
3861-
true);
3862-
b.projectPlus(
3863-
b.alias(tsEpochExpr, "__ts_epoch__"),
3864-
b.aggregateCall(SqlStdOperatorTable.MAX, tsEpochExpr).over().as("__max_epoch__"));
3865-
3866-
// Step 2: Compute period_number and offset
3867-
RexNode tsEpoch = b.field("__ts_epoch__");
3868-
RexNode maxEpoch = b.field("__max_epoch__");
3869-
RexNode spanLit = rx.makeBigintLiteral(BigDecimal.valueOf(spanSec));
3870-
3871-
// period = (max_epoch - ts_epoch) / span_sec + 1 (integer division truncates)
3872-
RexNode diff = rx.makeCall(SqlStdOperatorTable.MINUS, maxEpoch, tsEpoch);
3873-
RexNode periodNum =
3874-
rx.makeCall(
3875-
SqlStdOperatorTable.PLUS,
3876-
rx.makeCall(SqlStdOperatorTable.DIVIDE, diff, spanLit),
3877-
rx.makeExactLiteral(BigDecimal.ONE, bigintType));
3878-
3879-
// offset_sec = ts_epoch MOD span_sec
3880-
// Convert back to actual timestamp: latest_period_start + offset
3881-
RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MOD, tsEpoch, spanLit);
3882-
RexNode latestPeriodStart =
3883-
rx.makeCall(
3884-
SqlStdOperatorTable.MINUS,
3885-
maxEpoch,
3886-
rx.makeCall(SqlStdOperatorTable.MOD, maxEpoch, spanLit));
3887-
RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, latestPeriodStart, offsetSec);
3888-
RexNode displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch);
3889-
3890-
// Compute base_offset for absolute period naming in execution engine.
3891-
// align=now: reference = current time
3892-
// align=end: reference = WHERE upper bound (search end time), fallback to now
3850+
RexNode periodNum;
3851+
RexNode displayTimestamp;
38933852
RexNode baseOffset;
3894-
long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000;
3895-
Long referenceEpoch = null;
3896-
if ("end".equals(node.getAlign())) {
3897-
// Try to extract the upper bound from a WHERE clause on the timestamp field
3898-
referenceEpoch = extractTimestampUpperBound(node);
3899-
}
3900-
if (referenceEpoch == null) {
3901-
referenceEpoch = nowEpochSec;
3902-
}
3903-
RexNode refLit = rx.makeBigintLiteral(BigDecimal.valueOf(referenceEpoch));
3904-
baseOffset =
3905-
rx.makeCall(
3906-
SqlStdOperatorTable.DIVIDE,
3907-
rx.makeCall(SqlStdOperatorTable.MINUS, refLit, maxEpoch),
3908-
spanLit);
3853+
3854+
if (variableLength) {
3855+
// --- Variable-length units (month, quarter, year): EXTRACT-based calendar arithmetic ---
3856+
RexNode tsField = b.field(tsFieldName);
3857+
RexNode tsUnitNum =
3858+
TimewrapUtils.calendarUnitNumber(rx, tsField, node.getUnit(), node.getValue());
3859+
3860+
b.projectPlus(b.aggregateCall(SqlStdOperatorTable.MAX, tsField).over().as("__max_ts__"));
3861+
RexNode maxTs = b.field("__max_ts__");
3862+
RexNode maxUnitNum =
3863+
TimewrapUtils.calendarUnitNumber(rx, maxTs, node.getUnit(), node.getValue());
3864+
3865+
periodNum =
3866+
rx.makeCall(
3867+
SqlStdOperatorTable.PLUS,
3868+
rx.makeCall(SqlStdOperatorTable.MINUS, maxUnitNum, tsUnitNum),
3869+
rx.makeExactLiteral(BigDecimal.ONE, bigintType));
3870+
3871+
RexNode tsEpoch =
3872+
rx.makeCast(bigintType, rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, tsField), true);
3873+
RexNode unitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, tsField, node.getUnit());
3874+
RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MINUS, tsEpoch, unitStartEpoch);
3875+
RexNode maxUnitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, maxTs, node.getUnit());
3876+
RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, maxUnitStartEpoch, offsetSec);
3877+
displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch);
3878+
3879+
long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000;
3880+
Long referenceEpoch = null;
3881+
if ("end".equals(node.getAlign())) {
3882+
referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node);
3883+
}
3884+
if (referenceEpoch == null) {
3885+
referenceEpoch = nowEpochSec;
3886+
}
3887+
long refUnitNum =
3888+
TimewrapUtils.calendarUnitNumberFromEpoch(
3889+
referenceEpoch, node.getUnit(), node.getValue());
3890+
RexNode refUnitNumLit = rx.makeBigintLiteral(BigDecimal.valueOf(refUnitNum));
3891+
baseOffset = rx.makeCall(SqlStdOperatorTable.MINUS, refUnitNumLit, maxUnitNum);
3892+
3893+
} else {
3894+
// --- Fixed-length units (sec, min, hr, day, week): epoch-based arithmetic ---
3895+
long spanSec = TimewrapUtils.spanToSeconds(node.getUnit(), node.getValue());
3896+
3897+
RexNode tsEpochExpr =
3898+
rx.makeCast(
3899+
bigintType,
3900+
rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, b.field(tsFieldName)),
3901+
true);
3902+
b.projectPlus(
3903+
b.alias(tsEpochExpr, "__ts_epoch__"),
3904+
b.aggregateCall(SqlStdOperatorTable.MAX, tsEpochExpr).over().as("__max_epoch__"));
3905+
3906+
RexNode tsEpoch = b.field("__ts_epoch__");
3907+
RexNode maxEpoch = b.field("__max_epoch__");
3908+
RexNode spanLit = rx.makeBigintLiteral(BigDecimal.valueOf(spanSec));
3909+
3910+
RexNode diff = rx.makeCall(SqlStdOperatorTable.MINUS, maxEpoch, tsEpoch);
3911+
periodNum =
3912+
rx.makeCall(
3913+
SqlStdOperatorTable.PLUS,
3914+
rx.makeCall(SqlStdOperatorTable.DIVIDE, diff, spanLit),
3915+
rx.makeExactLiteral(BigDecimal.ONE, bigintType));
3916+
3917+
RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MOD, tsEpoch, spanLit);
3918+
RexNode latestPeriodStart =
3919+
rx.makeCall(
3920+
SqlStdOperatorTable.MINUS,
3921+
maxEpoch,
3922+
rx.makeCall(SqlStdOperatorTable.MOD, maxEpoch, spanLit));
3923+
RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, latestPeriodStart, offsetSec);
3924+
displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch);
3925+
3926+
long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000;
3927+
Long referenceEpoch = null;
3928+
if ("end".equals(node.getAlign())) {
3929+
referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node);
3930+
}
3931+
if (referenceEpoch == null) {
3932+
referenceEpoch = nowEpochSec;
3933+
}
3934+
RexNode refLit = rx.makeBigintLiteral(BigDecimal.valueOf(referenceEpoch));
3935+
baseOffset =
3936+
rx.makeCall(
3937+
SqlStdOperatorTable.DIVIDE,
3938+
rx.makeCall(SqlStdOperatorTable.MINUS, refLit, maxEpoch),
3939+
spanLit);
3940+
}
39093941

39103942
// Step 3: Project [display_timestamp, value_columns..., base_offset, period]
39113943
// base_offset is included in the group key so it survives the PIVOT
@@ -3925,8 +3957,8 @@ public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) {
39253957
b.groupKey(b.field(tsFieldName), b.field("__base_offset__")),
39263958
valueFieldNames.stream().map(f -> (RelBuilder.AggCall) b.max(b.field(f)).as("")).toList(),
39273959
ImmutableList.of(b.field("__period__")),
3928-
IntStream.rangeClosed(1, TIMEWRAP_MAX_PERIODS)
3929-
.map(i -> TIMEWRAP_MAX_PERIODS + 1 - i) // reverse: oldest period first
3960+
IntStream.rangeClosed(1, TimewrapUtils.MAX_PERIODS)
3961+
.map(i -> TimewrapUtils.MAX_PERIODS + 1 - i) // reverse: oldest period first
39303962
.mapToObj(
39313963
i ->
39323964
Map.entry(
@@ -3960,117 +3992,6 @@ public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) {
39603992
return b.peek();
39613993
}
39623994

3963-
/**
3964-
* Convert a span unit and value to approximate seconds. Variable-length units use standard
3965-
* approximations: month=30 days, quarter=91 days, year=365 days.
3966-
*/
3967-
private long timewrapSpanToSeconds(SpanUnit unit, int value) {
3968-
return switch (unit.getName()) {
3969-
case "s" -> value;
3970-
case "m" -> value * 60L;
3971-
case "h" -> value * 3_600L;
3972-
case "d" -> value * 86_400L;
3973-
case "w" -> value * 7L * 86_400L;
3974-
case "M" -> value * 30L * 86_400L; // month ≈ 30 days
3975-
case "q" -> value * 91L * 86_400L; // quarter ≈ 91 days
3976-
case "y" -> value * 365L * 86_400L; // year ≈ 365 days
3977-
default ->
3978-
throw new SemanticCheckException("Unsupported time unit in timewrap: " + unit.getName());
3979-
};
3980-
}
3981-
3982-
/**
3983-
* Get the timescale base name for timewrap column naming. Returns singular and plural forms
3984-
* separated by "|", e.g., "day|days". Used by the execution engine to build absolute period names
3985-
* like "501days_before".
3986-
*/
3987-
private String timewrapUnitBaseName(SpanUnit unit, int value) {
3988-
String singular =
3989-
switch (unit.getName()) {
3990-
case "s" -> "second";
3991-
case "m" -> "minute";
3992-
case "h" -> "hour";
3993-
case "d" -> "day";
3994-
case "w" -> "week";
3995-
case "M" -> "month";
3996-
case "q" -> "quarter";
3997-
case "y" -> "year";
3998-
default -> "period";
3999-
};
4000-
String plural = singular + "s";
4001-
// Encode value so execution engine can compute totalUnits = (base_offset + period) * value
4002-
return value + "|" + singular + "|" + plural;
4003-
}
4004-
4005-
/**
4006-
* Walk the AST from a Timewrap node to find a WHERE clause with an upper bound on the timestamp
4007-
* field (e.g., @timestamp <= '2024-07-03 18:00:00'). Returns the upper bound as epoch seconds, or
4008-
* null if not found.
4009-
*/
4010-
private Long extractTimestampUpperBound(Timewrap node) {
4011-
// Walk: Timewrap → Chart → Filter → inspect condition
4012-
Node current = node;
4013-
while (current != null && !current.getChild().isEmpty()) {
4014-
current = current.getChild().get(0);
4015-
if (current instanceof Filter filter) {
4016-
return findUpperBound(filter.getCondition());
4017-
}
4018-
}
4019-
return null;
4020-
}
4021-
4022-
/** Recursively search an expression tree for a timestamp upper bound (<=). */
4023-
private Long findUpperBound(UnresolvedExpression expr) {
4024-
if (expr instanceof And) {
4025-
And and = (And) expr;
4026-
Long left = findUpperBound(and.getLeft());
4027-
Long right = findUpperBound(and.getRight());
4028-
// If both sides have upper bounds, use the smaller one (tighter bound)
4029-
if (left != null && right != null) return Math.min(left, right);
4030-
return left != null ? left : right;
4031-
}
4032-
if (expr instanceof Compare cmp) {
4033-
String op = cmp.getOperator();
4034-
// Check for @timestamp <= X or @timestamp < X
4035-
if (("<=".equals(op) || "<".equals(op)) && isTimestampField(cmp.getLeft())) {
4036-
return parseTimestampLiteral(cmp.getRight());
4037-
}
4038-
// Check for X >= @timestamp or X > @timestamp
4039-
if ((">=".equals(op) || ">".equals(op)) && isTimestampField(cmp.getRight())) {
4040-
return parseTimestampLiteral(cmp.getLeft());
4041-
}
4042-
}
4043-
return null;
4044-
}
4045-
4046-
private boolean isTimestampField(UnresolvedExpression expr) {
4047-
if (expr instanceof Field field) {
4048-
String name = field.getField().toString();
4049-
return "@timestamp".equals(name) || "timestamp".equals(name);
4050-
}
4051-
return false;
4052-
}
4053-
4054-
private Long parseTimestampLiteral(UnresolvedExpression expr) {
4055-
if (expr instanceof Literal lit && lit.getValue() instanceof String s) {
4056-
try {
4057-
// Parse "yyyy-MM-dd HH:mm:ss" format
4058-
java.time.LocalDateTime ldt =
4059-
java.time.LocalDateTime.parse(
4060-
s, java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
4061-
return ldt.toEpochSecond(java.time.ZoneOffset.UTC);
4062-
} catch (Exception e) {
4063-
// Try ISO format
4064-
try {
4065-
return java.time.Instant.parse(s).getEpochSecond();
4066-
} catch (Exception ignored) {
4067-
return null;
4068-
}
4069-
}
4070-
}
4071-
return null;
4072-
}
4073-
40743995
/**
40753996
* Aggregate by column split then rank by grand total (summed value of each category). The output
40763997
* is <code>[col-split, grand-total, row-number]</code>

0 commit comments

Comments
 (0)