Merge upstream/main into PR #5397 catch-up branch

Bring the analytics-engine catch-up PR up to current upstream/main by
resolving conflicts introduced by 4 main commits since 2026-04-30:

- #5419 (LENGTH/REGEXP_REPLACE/DATE_TRUNC unified function spec)
- #5408 (datetime type normalization)
- #5414 (Gradle wrapper bump + @ignore exclusion)
- #5399 (FGAC-scoped SQL cursor continuation)

Resolutions:

api/spec/* (LanguageSpec, UnifiedFunctionSpec, UnifiedPplSpec,
UnifiedSqlSpec): took main. Main is a strict superset — adds
postAnalysisRules and preCompilationRules extension points, the new
FunctionSpecBuilder DSL, SCALAR category for length/regexp_replace/
date_trunc, the DatetimeExtension on PPL spec, and the CoreExtension
wiring on SQL spec. PR's RELEVANCE category is preserved unchanged.

api/UnifiedQueryPlanner.java, api/compiler/UnifiedQueryCompiler.java:
took main. Both adopt the new postAnalysisRules / preCompilationRules
hooks introduced in #5408 / #5419.

core/executor/QueryService.java: composed both sides — kept HEAD's
CalciteClassLoaderHelper.withCalciteClassLoader wrapper around main's
StageErrorHandler stage tracking. Same pattern as the original PR
resolution; both improvements are orthogonal.

plugin/SQLPlugin.java: kept HEAD imports for ExplainResponse and
QueryType (referenced by createSqlAnalyticsRouter, which only exists
on the feature branch).

plugin/transport/TransportPPLQueryAction.java: kept HEAD's
queryPlanExecutor parameter alongside main's extensionsHolder
parameter, since both are referenced in the constructor body.

legacy/plugin/RestSqlAction.java: took HEAD. The 3-way merge produced
a duplicated handleException/getRawErrorCode block; HEAD already
contained both the delegateToV2Engine refactor and the ErrorReport
unwrap from main, so HEAD is the correct superset.

Build: :api, :core, :opensearch-sql-plugin, :legacy compileJava and
unit tests pass; spotlessCheck passes.

Signed-off-by: Kai Huang <ahkcs@amazon.com>

Author: ahkcs

api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java

@@ -60,7 +60,15 @@ public UnifiedQueryPlanner(UnifiedQueryContext context) {
    */
   public RelNode plan(String query) {
     try {
-      return context.measure(ANALYZE, () -> strategy.plan(query));
+      return context.measure(
+          ANALYZE,
+          () -> {
+            RelNode plan = strategy.plan(query);
+            for (var shuttle : context.getLangSpec().postAnalysisRules()) {
+              plan = plan.accept(shuttle);
+            }
+            return plan;
+          });
     } catch (SyntaxCheckException | UnsupportedOperationException e) {
       throw e;
     } catch (Exception e) {

api/src/main/java/org/opensearch/sql/api/compiler/UnifiedQueryCompiler.java

@@ -55,6 +55,11 @@ public PreparedStatement compile(@NonNull RelNode plan) {
   }
 
   private PreparedStatement doCompile(RelNode plan) throws Exception {
+    // Apply pre-compilation rules (e.g., late-binding function impl)
+    for (var rule : context.getLangSpec().preCompilationRules()) {
+      plan = plan.accept(rule);
+    }
+
     // Apply shuttle to convert LogicalTableScan to BindableTableScan
     final RelHomogeneousShuttle shuttle =
         new RelHomogeneousShuttle() {

api/src/main/java/org/opensearch/sql/api/spec/FunctionSpecBuilder.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.api.spec;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.function.BiFunction;
+import javax.annotation.Nullable;
+import lombok.RequiredArgsConstructor;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperandCountRange;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.InferTypes;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlOperandMetadata;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+
+/** Fluent DSL for building {@link UnifiedFunctionSpec} instances. */
+@RequiredArgsConstructor
+class FunctionSpecBuilder {
+  /** Function name to register. */
+  private final String name;
+
+  /**
+   * Wraps an existing Calcite operator, preserving its native type system and RexImpTable
+   * implementation for in-memory execution.
+   *
+   * @param op the Calcite operator to delegate to
+   * @return a builder that produces the spec on {@code build()}
+   */
+  DelegateFunctionBuilder delegateTo(SqlOperator op) {
+    return new DelegateFunctionBuilder(name, op);
+  }
+
+  /**
+   * Builds a pushdown-only UDF with relaxed type checking. The resulting function has no local
+   * implementation and delegates execution to the data source via pushdown.
+   *
+   * @param paramNames required parameter names for signature display
+   * @return a builder that produces the spec on {@code build()}
+   */
+  CatalogFunctionBuilder vararg(String... paramNames) {
+    return new CatalogFunctionBuilder(name, List.of(paramNames));
+  }
+
+  /**
+   * Builds a typed SqlFunction with strict operand type checking. Optionally accepts a late-binding
+   * {@code impl} that rewrites the function into executable Calcite expressions at compilation
+   * time.
+   *
+   * @param families operand type families for validation
+   * @return a builder that produces the spec on {@code build()}
+   */
+  DefaultFunctionBuilder operands(SqlTypeFamily... families) {
+    return new DefaultFunctionBuilder(name, families);
+  }
+
+  @RequiredArgsConstructor
+  static class DefaultFunctionBuilder {
+    private final String name;
+    private final SqlTypeFamily[] operandFamilies;
+    private SqlReturnTypeInference returnType;
+    private SqlFunctionCategory category = SqlFunctionCategory.USER_DEFINED_FUNCTION;
+    private @Nullable BiFunction<RexBuilder, RexCall, RexNode> impl;
+
+    DefaultFunctionBuilder returns(SqlReturnTypeInference type) {
+      this.returnType = type;
+      return this;
+    }
+
+    DefaultFunctionBuilder category(SqlFunctionCategory cat) {
+      this.category = cat;
+      return this;
+    }
+
+    /**
+     * Defines how this function executes by rewriting to existing Calcite operators. Applied only
+     * at compilation time (late binding) — the logical plan preserves the original function call.
+     *
+     * @param impl rewrite function that converts this call into executable RexNodes
+     * @return this builder
+     */
+    DefaultFunctionBuilder impl(BiFunction<RexBuilder, RexCall, RexNode> impl) {
+      this.impl = impl;
+      return this;
+    }
+
+    UnifiedFunctionSpec build() {
+      Objects.requireNonNull(returnType, "returns() is required");
+      SqlFunction op =
+          new SqlFunction(
+              name.toUpperCase(),
+              SqlKind.OTHER_FUNCTION,
+              returnType,
+              null,
+              OperandTypes.family(operandFamilies),
+              category);
+      return new UnifiedFunctionSpec(name.toLowerCase(), op, impl);
+    }
+  }
+
+  @RequiredArgsConstructor
+  static class DelegateFunctionBuilder {
+    private final String name;
+    private final SqlOperator operator;
+
+    UnifiedFunctionSpec build() {
+      return new UnifiedFunctionSpec(name.toLowerCase(), operator, null);
+    }
+  }
+
+  @RequiredArgsConstructor
+  static class CatalogFunctionBuilder {
+    private final String name;
+    private final List<String> paramNames;
+    private SqlReturnTypeInference returnType;
+
+    CatalogFunctionBuilder returnType(SqlReturnTypeInference type) {
+      this.returnType = type;
+      return this;
+    }
+
+    UnifiedFunctionSpec build() {
+      Objects.requireNonNull(returnType, "returnType is required");
+      return new UnifiedFunctionSpec(
+          name,
+          new SqlUserDefinedFunction(
+              new SqlIdentifier(name, SqlParserPos.ZERO),
+              SqlKind.OTHER_FUNCTION,
+              returnType,
+              InferTypes.ANY_NULLABLE,
+              new VariadicOperandMetadata(paramNames),
+              List::of), // Pushdown-only: no local implementation
+          null);
+    }
+  }
+
+  /**
+   * Custom operand metadata that bypasses Calcite's built-in type checking. Calcite's {@code
+   * FamilyOperandTypeChecker} rejects variadic calls (CALCITE-5366), so this implementation accepts
+   * any operand types and delegates validation to pushdown.
+   */
+  record VariadicOperandMetadata(List<String> paramNames) implements SqlOperandMetadata {
+
+    @Override
+    public List<String> paramNames() {
+      return paramNames;
+    }
+
+    @Override
+    public List<RelDataType> paramTypes(RelDataTypeFactory tf) {
+      return List.of();
+    }
+
+    @Override
+    public boolean checkOperandTypes(SqlCallBinding binding, boolean throwOnFailure) {
+      return true;
+    }
+
+    @Override
+    public SqlOperandCountRange getOperandCountRange() {
+      return SqlOperandCountRanges.from(paramNames.size());
+    }
+
+    @Override
+    public String getAllowedSignatures(SqlOperator op, String opName) {
+      return opName + "(" + String.join(", ", paramNames) + "[, option=value ...])";
+    }
+  }
+}

api/src/main/java/org/opensearch/sql/api/spec/LanguageSpec.java

@@ -7,6 +7,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.calcite.rel.RelShuttle;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlOperatorTable;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
@@ -17,8 +18,8 @@
 
 /**
  * Language specification defining the dialect the engine accepts. Provides parser configuration,
- * validator configuration, and composable {@link LanguageExtension}s that contribute operators and
- * post-parse rewrite rules.
+ * validator configuration, and composable {@link LanguageExtension}s that contribute operators,
+ * post-parse rewrite rules, and post-analysis rewrite rules.
  *
  * <p>Implementations define a complete language surface — for example, {@link UnifiedSqlSpec}
  * provides ANSI and extended SQL modes. A future PPL spec would implement this same interface once
@@ -27,8 +28,9 @@
 public interface LanguageSpec {
 
   /**
-   * A composable language extension that contributes operators and post-parse rewrite rules. All
-   * methods have defaults so extensions only override what they need.
+   * A composable language extension that contributes operators, post-parse rewrite rules, and
+   * post-analysis rewrite rules. All methods have defaults so extensions only override what they
+   * need.
    */
   interface LanguageExtension {
 
@@ -47,6 +49,23 @@ default SqlOperatorTable operators() {
     default List<SqlVisitor<SqlNode>> postParseRules() {
       return List.of();
     }
+
+    /**
+     * RelNode rewrite rules applied after analysis and before execution. Each rule transforms the
+     * logical plan tree. Rules within a single extension are applied in list order.
+     */
+    default List<RelShuttle> postAnalysisRules() {
+      return List.of();
+    }
+
+    /**
+     * Pre-compilation rules applied only before in-memory execution. Each rule transforms the
+     * logical plan (e.g., binding function implementations). Not applied when the plan is consumed
+     * by external engines.
+     */
+    default List<RelShuttle> preCompilationRules() {
+      return List.of();
+    }
   }
 
   /**
@@ -62,9 +81,9 @@ default List<SqlVisitor<SqlNode>> postParseRules() {
   SqlValidator.Config validatorConfig();
 
   /**
-   * Language extensions registered with this spec. Each extension contributes operators and
-   * post-parse rewrite rules that are composed by {@link #operatorTable()} and {@link
-   * #postParseRules()}.
+   * Language extensions registered with this spec. Each extension contributes operators, post-parse
+   * rewrite rules, and post-analysis rewrite rules composed by {@link #operatorTable()}, {@link
+   * #postParseRules()}, and {@link #postAnalysisRules()}.
    */
   List<LanguageExtension> extensions();
 
@@ -86,4 +105,20 @@ default SqlOperatorTable operatorTable() {
   default List<SqlVisitor<SqlNode>> postParseRules() {
     return extensions().stream().flatMap(ext -> ext.postParseRules().stream()).toList();
   }
+
+  /**
+   * All post-analysis RelNode rewrite rules from registered extensions, flattened in registration
+   * order. Applied to the logical plan after analysis and before execution.
+   */
+  default List<RelShuttle> postAnalysisRules() {
+    return extensions().stream().flatMap(ext -> ext.postAnalysisRules().stream()).toList();
+  }
+
+  /**
+   * All pre-compilation rules from registered extensions, flattened in registration order. Applied
+   * only before in-memory execution.
+   */
+  default List<RelShuttle> preCompilationRules() {
+    return extensions().stream().flatMap(ext -> ext.preCompilationRules().stream()).toList();
+  }
 }