[BugFix] Return all columns (struct and nested fields) listed when using head (#5518)

Signed-off-by: Du Tran <quangdutran809@gmail.com>

Author: quangdutran

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -516,8 +516,9 @@ private RelNode handleAllFieldsProject(Project node, CalcitePlanContext context)
           "Invalid field exclusion: operation would exclude all fields from the result set");
     }
     AllFields allFields = (AllFields) node.getProjectList().getFirst();
-    if (!(allFields instanceof AllFieldsExcludeMeta)) {
-      // Should not remove nested fields for AllFieldsExcludeMeta.
+    if (!(allFields instanceof AllFieldsExcludeMeta) && !context.isProjectVisited()) {
+      // Should not remove nested fields for AllFieldsExcludeMeta
+      // and when no explicit project has already curated the schema
       tryToRemoveNestedFields(context);
     }
     tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta);

docs/user/ppl/cmd/fields.md

@@ -198,8 +198,8 @@ fetched rows / total rows = 3/3
 
 ## Example 8: Selecting all fields
 
-The following query selects all fields defined in the index schema using `` `*` ``. Fields with null values are included in the result set:
-  
+The following query selects all fields defined in the index schema using `` `*` ``. Struct fields are returned alongside their flattened sub-fields, and fields with null values are included in the result set:
+
 ```ppl
 source=otellogs
 | where severityText = 'WARN'
@@ -211,11 +211,11 @@ The query returns the following results:
 
 ```text
 fetched rows / total rows = 1/1
-+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+
-| spanId   | traceId          | @timestamp          | instrumentationScope                                                                                                                      | severityText | resource                                                                                                                             | flags | attributes | droppedAttributesCount | severityNumber | time                | body                                                                                   |
-|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------|
-| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | WARN         | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | 0     | {}         | 0                      | 13             | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms |
-+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+
++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+
+| spanId   | traceId          | @timestamp          | instrumentationScope                                                                                                                      | instrumentationScope.droppedAttributesCount | instrumentationScope.name                                                   | instrumentationScope.version | severityText | resource                                                                                                                             | resource.attributes                                                                     | resource.attributes.host              | resource.attributes.host.name | resource.attributes.service | resource.attributes.service.name | resource.droppedAttributesCount | flags | attributes | droppedAttributesCount | severityNumber | time                | body                                                                                   |
+|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------|
+| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | 0                                           | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc | 0.49.0                       | WARN         | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}} | {'name': 'productcatalog-7c9d-zn4p2'} | productcatalog-7c9d-zn4p2     | {'name': 'product-catalog'} | product-catalog                  | 0                               | 0     | {}         | 0                      | 13             | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms |
++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+
 ```
 
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java

@@ -6,6 +6,7 @@
 package org.opensearch.sql.calcite.remote;
 
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TELEMETRY;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
@@ -29,6 +30,7 @@ public void init() throws Exception {
 
     loadIndex(Index.BANK);
     loadIndex(Index.TELEMETRY);
+    loadIndex(Index.DEEP_NESTED);
 
     // Pre-create test_eval through the helper so the analytics-engine compatibility run
     // (tests.analytics.parquet_indices=true) provisions it as a parquet-backed composite
@@ -160,7 +162,7 @@ public void testEvalDottedNamePreservesStructParent_ImplicitProject() throws IOE
     // longer dropped by `shouldOverrideField`'s prefix branch.
     JSONObject result =
         executeQuery("source=test_eval_agent | fields agent | eval `agent.name` = 'test'");
-    verifySchema(result, schema("agent", "struct"));
+    verifySchema(result, schema("agent", "struct"), schema("agent.name", "string"));
   }
 
   @Test
@@ -233,4 +235,12 @@ public void testEvalStringConcatenationWithExistingData() throws IOException {
         rows("Hattie", "Bond", "Hattie Bond"),
         rows("Nanette", "Bates", "Nanette Bates"));
   }
+
+  @Test
+  public void testStruckFieldAndSubFieldWithHead() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format("source=%s | fields city.name, city | head", TEST_INDEX_DEEP_NESTED));
+    verifySchema(result, schema("city.name", "string"), schema("city", "struct"));
+  }
 }