Add an ignored test case and limitation explainations for flatten after fields

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

docs/user/ppl/cmd/flatten.rst

@@ -106,6 +106,12 @@ PPL query::
 
 Limitations
 ===========
+* ``flatten`` after ``fields`` is not supported under the current
+  implementation.
+  | E.g. ``source=my-index | fields message | flatten message`` will not work.
+  This is because the current implementation relies on the flattened fields
+  like ``message.info``, ``message.author``, etc. to be present in the read
+  result.
 * The command works only with Calcite enabled. This can be set with the
   following command:
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFlattenCommandIT.java

@@ -19,6 +19,7 @@
 import org.hamcrest.Matcher;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.junit.Ignore;
 import org.junit.jupiter.api.Test;
 import org.opensearch.client.Request;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
@@ -124,6 +125,33 @@ public void testFlattenNullField() throws IOException {
     client().performRequest(deleteRequest);
   }
 
+  // TODO: Enable after fixing issue #3459 and #3751
+  //  https://github.com/opensearch-project/sql/issues/3459
+  //  https://github.com/opensearch-project/sql/issues/3751
+  @Ignore
+  @Test
+  public void testFlattenAfterFields() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | where myNum=1 | fields message | flatten message",
+                TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS));
+    verifySchema(
+        result,
+        schema("message", "array"),
+        schema("author", "string"),
+        schema("dayOfWeek", "bigint"),
+        schema("info", "string"));
+    verifyDataRows(
+        result,
+        rows(
+            new JSONArray()
+                .put(new JSONObject().put("info", "a").put("author", "e").put("dayOfWeek", 1)),
+            "e",
+            1,
+            "a"));
+  }
+
   @SuppressWarnings("unchecked")
   private static Matcher<JSONArray>[] getExpectedRows() {
     return new org.hamcrest.TypeSafeMatcher[] {

ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFlattenTest.java

@@ -49,7 +49,7 @@ protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpec
     // Add an empty table with name DEPT for test purpose
     ImmutableList<Object[]> rows =
         ImmutableList.of(
-            new Object[] {10, ImmutableList.of(7369, "ALLEN"), "SMITH", 7369},
+            new Object[] {10, ImmutableList.of(7369, "SMITH"), "SMITH", 7369},
             new Object[] {20, ImmutableList.of(7499, "ALLEN"), "ALLEN", 7499},
             new Object[] {30, ImmutableList.of(7521, "WARD"), "WARD", 7521});
     schema.add("DEPT", new TableWithStruct(rows));
@@ -73,7 +73,7 @@ public void testFlatten() {
         "SELECT `DEPTNO`, `EMP`, `EMP.EMPNAME` `EMPNAME`, `EMP.EMPNO` `EMPNO`\nFROM `scott`.`DEPT`";
     verifyPPLToSparkSQL(root, expectedSparkSql);
     String expectedResult =
-        "DEPTNO=10; EMP={7369, ALLEN}; EMPNAME=SMITH; EMPNO=7369\n"
+        "DEPTNO=10; EMP={7369, SMITH}; EMPNAME=SMITH; EMPNO=7369\n"
             + "DEPTNO=20; EMP={7499, ALLEN}; EMPNAME=ALLEN; EMPNO=7499\n"
             + "DEPTNO=30; EMP={7521, WARD}; EMPNAME=WARD; EMPNO=7521\n";
     verifyResult(root, expectedResult);
@@ -91,7 +91,7 @@ public void testFlattenWithAliases() {
         "SELECT `DEPTNO`, `EMP`, `EMP.EMPNAME` `name`, `EMP.EMPNO` `number`\nFROM `scott`.`DEPT`";
     verifyPPLToSparkSQL(root, expectedSparkSql);
     String expectedResult =
-        "DEPTNO=10; EMP={7369, ALLEN}; name=SMITH; number=7369\n"
+        "DEPTNO=10; EMP={7369, SMITH}; name=SMITH; number=7369\n"
             + "DEPTNO=20; EMP={7499, ALLEN}; name=ALLEN; number=7499\n"
             + "DEPTNO=30; EMP={7521, WARD}; name=WARD; number=7521\n";
     verifyResult(root, expectedResult);
@@ -109,7 +109,7 @@ public void testProject() {
         "LogicalProject(DEPTNO=[$0], EMP=[$1])\n  LogicalTableScan(table=[[scott, DEPT]])\n";
     verifyLogical(root, expectedLogical);
     String expectedResult =
-        "DEPTNO=10; EMP={7369, ALLEN}\n"
+        "DEPTNO=10; EMP={7369, SMITH}\n"
             + "DEPTNO=20; EMP={7499, ALLEN}\n"
             + "DEPTNO=30; EMP={7521, WARD}\n";
     verifyResult(root, expectedResult);