[GRAMMAR REMOVAL] Internally handle highlight instead of expose as a command

Signed-off-by: Jialiang Liang <jiallian@amazon.com>

Author: RyanL1997

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -61,6 +61,7 @@
 import org.opensearch.sql.ast.tree.Filter;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Highlight;
+import org.opensearch.sql.ast.tree.HighlightConfig;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.MinSpanBin;
 import org.opensearch.sql.ast.tree.MvCombine;
@@ -585,8 +586,12 @@ public static Trendline trendline(
     return new Trendline(sortField, Arrays.asList(computations)).attach(input);
   }
 
-  public static Highlight highlight(UnresolvedPlan input, List<String> highlightArgs) {
-    return new Highlight(highlightArgs).attach(input);
+  public static Highlight highlight(UnresolvedPlan input, HighlightConfig highlightConfig) {
+    return new Highlight(highlightConfig).attach(input);
+  }
+
+  public static Highlight highlight(UnresolvedPlan input, List<String> highlightFields) {
+    return new Highlight(new HighlightConfig(highlightFields)).attach(input);
   }
 
   public static AppendPipe appendPipe(UnresolvedPlan input, UnresolvedPlan subquery) {

core/src/main/java/org/opensearch/sql/ast/tree/Highlight.java

@@ -21,7 +21,7 @@
 public class Highlight extends UnresolvedPlan {
 
   private UnresolvedPlan child;
-  private final List<String> highlightArgs;
+  private final HighlightConfig highlightConfig;
 
   @Override
   public Highlight attach(UnresolvedPlan child) {

core/src/main/java/org/opensearch/sql/ast/tree/HighlightConfig.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import java.util.List;
+
+/**
+ * Bundles highlight configuration: field names (or wildcards) and optional pre/post tags and
+ * fragment size. Supports both the simple array format ({@code ["*"]}) and the rich OSD object
+ * format with {@code pre_tags}, {@code post_tags}, {@code fields}, and {@code fragment_size}.
+ */
+public record HighlightConfig(
+    List<String> fields, List<String> preTags, List<String> postTags, Integer fragmentSize) {
+
+  /** Convenience constructor for the simple array format (fields only, no tag/size overrides). */
+  public HighlightConfig(List<String> fields) {
+    this(fields, null, null, null);
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java

@@ -22,6 +22,7 @@
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.tools.FrameworkConfig;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.ast.tree.HighlightConfig;
 import org.opensearch.sql.calcite.utils.CalciteToolsHelper;
 import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelBuilder;
 import org.opensearch.sql.common.setting.Settings;
@@ -45,6 +46,7 @@ public class CalcitePlanContext {
   private static final ThreadLocal<Boolean> legacyPreferredFlag =
       ThreadLocal.withInitial(() -> true);
 
+  @Getter @Setter private HighlightConfig highlightConfig;
   @Getter @Setter private boolean isResolvingJoinCondition = false;
   @Getter @Setter private boolean isResolvingSubquery = false;
   @Getter @Setter private boolean inCoalesceFunction = false;

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -230,9 +230,18 @@ public RelNode visitRelation(Relation node, CalcitePlanContext context) {
     RelNode scan = context.relBuilder.peek();
 
     if (scan instanceof AliasFieldsWrappable) {
-      return ((AliasFieldsWrappable) scan).wrapProjectForAliasFields(context.relBuilder);
+      ((AliasFieldsWrappable) scan).wrapProjectForAliasFields(context.relBuilder);
     }
-    return scan;
+
+    // Wrap with LogicalHighlight if highlight config is set on context (from API request)
+    if (context.getHighlightConfig() != null) {
+      RelNode input = context.relBuilder.build();
+      LogicalHighlight highlight = LogicalHighlight.create(input, context.getHighlightConfig());
+      context.relBuilder.push(highlight);
+      context.setHighlightConfig(null); // Clear for join safety
+    }
+
+    return context.relBuilder.peek();
   }
 
   // This is a tool method to add an existed RelOptTable to builder stack, not used for now
@@ -3192,11 +3201,9 @@ static ChartConfig fromArguments(ArgumentMap argMap) {
 
   @Override
   public RelNode visitHighlight(Highlight node, CalcitePlanContext context) {
+    context.setHighlightConfig(node.getHighlightConfig());
     visitChildren(node, context);
-    RelNode input = context.relBuilder.build();
-    LogicalHighlight highlight = LogicalHighlight.create(input, node.getHighlightArgs());
-    context.relBuilder.push(highlight);
-    return highlight;
+    return context.relBuilder.peek();
   }
 
   @Override

core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalHighlight.java

@@ -16,31 +16,32 @@
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.opensearch.sql.ast.tree.HighlightConfig;
 import org.opensearch.sql.expression.HighlightExpression;
 
 /**
  * Logical relational node representing a PPL {@code | highlight} command. Stores the highlight
- * arguments (terms to highlight or {@code *} for wildcard) and adds a {@code _highlight} column to
- * the output row type. An optimizer rule ({@code HighlightIndexScanRule}) pushes this node down
- * into the OpenSearch index scan.
+ * configuration (fields, pre/post tags, fragment size) and adds a {@code _highlight} column to the
+ * output row type. An optimizer rule ({@code HighlightIndexScanRule}) pushes this node down into
+ * the OpenSearch index scan.
  */
 public class LogicalHighlight extends SingleRel {
 
-  @Getter private final List<String> highlightArgs;
+  @Getter private final HighlightConfig highlightConfig;
   private final RelDataType highlightRowType;
 
   protected LogicalHighlight(
       RelOptCluster cluster,
       RelTraitSet traitSet,
       RelNode input,
-      List<String> highlightArgs,
+      HighlightConfig highlightConfig,
       RelDataType highlightRowType) {
     super(cluster, traitSet, input);
-    this.highlightArgs = highlightArgs;
+    this.highlightConfig = highlightConfig;
     this.highlightRowType = highlightRowType;
   }
 
-  public static LogicalHighlight create(RelNode input, List<String> highlightArgs) {
+  public static LogicalHighlight create(RelNode input, HighlightConfig highlightConfig) {
     final RelOptCluster cluster = input.getCluster();
     RelTraitSet traitSet = cluster.traitSetOf(Convention.NONE);
 
@@ -54,7 +55,7 @@ public static LogicalHighlight create(RelNode input, List<String> highlightArgs)
           HighlightExpression.HIGHLIGHT_FIELD, typeFactory.createSqlType(SqlTypeName.ANY));
     }
 
-    return new LogicalHighlight(cluster, traitSet, input, highlightArgs, schemaBuilder.build());
+    return new LogicalHighlight(cluster, traitSet, input, highlightConfig, schemaBuilder.build());
   }
 
   @Override
@@ -66,11 +67,11 @@ protected RelDataType deriveRowType() {
   public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
     assert traitSet.containsIfApplicable(Convention.NONE);
     return new LogicalHighlight(
-        getCluster(), traitSet, sole(inputs), highlightArgs, highlightRowType);
+        getCluster(), traitSet, sole(inputs), highlightConfig, highlightRowType);
   }
 
   @Override
   public RelWriter explainTerms(RelWriter pw) {
-    return super.explainTerms(pw).item("highlightArgs", highlightArgs);
+    return super.explainTerms(pw).item("highlightConfig", highlightConfig.fields());
   }
 }

docs/category.json

@@ -22,7 +22,6 @@
     "user/ppl/cmd/fillnull.md",
     "user/ppl/cmd/grok.md",
     "user/ppl/cmd/head.md",
-    "user/ppl/cmd/highlight.md",
     "user/ppl/cmd/join.md",
     "user/ppl/cmd/lookup.md",
     "user/ppl/cmd/mvcombine.md",

docs/user/ppl/cmd/highlight.md

@@ -201,3 +201,55 @@ Expected output (trimmed):
 - Plan node names use Calcite physical operator names (for example, `EnumerableCalc` or `CalciteEnumerableIndexScan`).
 - Plan `time_ms` is inclusive of child operators and represents wall-clock time; overlapping work can make summed plan times exceed `summary.total_time_ms`.
 - Scan nodes reflect operator wall-clock time; background prefetch can make scan time smaller than total request latency.
+
+## Highlight
+
+You can add a `highlight` parameter to the PPL request body to enable search-result highlighting. When enabled, the response includes a `_highlight` field containing matching fragments with the specified tags.
+
+Two formats are supported:
+
+### Simple array format
+
+Pass a JSON array of field names or wildcards. Use `["*"]` to highlight all fields that match the search query.
+
+```bash ppl
+curl -sS -H 'Content-Type: application/json' \
+  -X POST localhost:9200/_plugins/_ppl \
+  -d '{
+        "query": "source=accounts \"Holmes\"",
+        "highlight": ["*"]
+      }'
+```
+
+### Object format (OpenSearch Dashboards)
+
+Pass a JSON object with `fields`, `pre_tags`, `post_tags`, and `fragment_size`. This is the format used by OpenSearch Dashboards.
+
+```bash ppl
+curl -sS -H 'Content-Type: application/json' \
+  -X POST localhost:9200/_plugins/_ppl \
+  -d '{
+        "query": "source=accounts \"Holmes\"",
+        "highlight": {
+          "pre_tags": ["@opensearch-dashboards-highlighted-field@"],
+          "post_tags": ["@/opensearch-dashboards-highlighted-field@"],
+          "fields": {"*": {}},
+          "fragment_size": 2147483647
+        }
+      }'
+```
+
+### Parameters
+
+| Parameter       | Type            | Required | Description                                                                                                  |
+|-----------------|-----------------|----------|--------------------------------------------------------------------------------------------------------------|
+| `fields`        | Object          | Yes      | An object whose keys are field names or wildcards (e.g. `{"*": {}}`) to highlight.                           |
+| `pre_tags`      | Array of string | No       | Tags inserted before highlighted tokens. Defaults to `<em>`.                                                 |
+| `post_tags`     | Array of string | No       | Tags inserted after highlighted tokens. Defaults to `</em>`.                                                 |
+| `fragment_size` | Integer         | No       | Maximum character size of a highlight fragment. Defaults to `2147483647` (returns the full field value).      |
+
+### Notes
+
+- Highlighting requires a search query in the PPL statement (e.g. `source=accounts "Holmes"`). Without a query, the `_highlight` field will be empty.
+- In the simple array format, `["*"]` highlights all fields. Specific terms like `["error", "login"]` highlight those terms across all fields.
+- In the object format, only the keys of the `fields` object are used; per-field options inside the value objects are currently ignored.