peng - hiden col solution 2

Signed-off-by: Jialiang Liang <jiallian@amazon.com>

Author: RyanL1997

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -23,6 +23,7 @@
 import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation;
 import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall;
 import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild;
+import static org.opensearch.sql.expression.HighlightExpression.HIGHLIGHT_FIELD;
 import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME;
 
 import com.google.common.base.Strings;
@@ -407,6 +408,15 @@ public RelNode visitProject(Project node, CalcitePlanContext context) {
       if (!context.isResolvingSubquery()) {
         context.setProjectVisited(true);
       }
+      // When highlight is active, include _highlight in the projection so it survives
+      // through the Calcite pipeline. This matches DSL behavior where _source filtering
+      // does not affect highlights.
+      if (CalcitePlanContext.getHighlightConfig() != null) {
+        int hlIndex = currentFields.indexOf(HIGHLIGHT_FIELD);
+        if (hlIndex >= 0) {
+          expandedFields.add(context.relBuilder.field(hlIndex));
+        }
+      }
       context.relBuilder.project(expandedFields);
     }
     return context.relBuilder.peek();

docs/user/ppl/interfaces/endpoint.md

@@ -155,11 +155,12 @@ Only the `address` field is highlighted. Rows where "Holmes" appears in other fi
 - Entries are `null` when a row has no highlight data for the requested fields.
 - The `highlights` array is **omitted entirely** when no `highlight` config is provided in the request (backward compatible).
 
-### Notes
+### Limitations
 
 - Highlighting is supported only in the Calcite engine.
 - The backend forwards the highlight config as-is to OpenSearch. The same highlighting behavior and limitations as [OpenSearch's highlighting API](https://opensearch.org/docs/latest/search-plugins/searching-data/highlight/) apply.
 - Piped commands (`where`, `sort`, `head`, `dedup`) narrow or reorder the result set but do not affect which terms are highlighted.
+- Highlighting works with **single-source queries only**. Joins (`| join`), subqueries, and multi-source queries are not supported — the `highlight` config is ignored in these cases.
 
 ## Explain
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLHighlightIT.java

@@ -43,11 +43,15 @@ public void testHighlightWithWildcardFields() throws IOException {
     boolean foundHighlight = false;
     for (int i = 0; i < highlights.length(); i++) {
       if (!highlights.isNull(i)) {
-        String hlStr = highlights.get(i).toString();
-        if (hlStr.contains("<em>Street</em>")) {
-          foundHighlight = true;
-          break;
+        JSONObject hl = highlights.getJSONObject(i);
+        for (String key : hl.keySet()) {
+          String fragment = hl.getJSONArray(key).getString(0);
+          if (fragment.contains("<em>") && fragment.contains("Street")) {
+            foundHighlight = true;
+            break;
+          }
         }
+        if (foundHighlight) break;
       }
     }
     assertTrue("Expected at least one highlight with <em> tags", foundHighlight);
@@ -87,11 +91,15 @@ public void testHighlightWithCustomTags() throws IOException {
     boolean foundCustomTag = false;
     for (int i = 0; i < highlights.length(); i++) {
       if (!highlights.isNull(i)) {
-        String hlStr = highlights.get(i).toString();
-        if (hlStr.contains("<mark>")) {
-          foundCustomTag = true;
-          break;
+        JSONObject hl = highlights.getJSONObject(i);
+        for (String key : hl.keySet()) {
+          String fragment = hl.getJSONArray(key).getString(0);
+          if (fragment.contains("<mark>")) {
+            foundCustomTag = true;
+            break;
+          }
         }
+        if (foundCustomTag) break;
       }
     }
     assertTrue("Expected custom <mark> tags in highlights", foundCustomTag);