[BugFix] Fix rename with wildcard applying on hidden fields (opensearch-project#5099) (opensearch-project#5350)

* [BugFix] Fix rename with wildcard applying on hidden/metadata fields (opensearch-project#5099)

Signed-off-by: Songkan Tang <songkant@amazon.com>

* Fix test assertion for rename wildcard excluding hidden/metadata fields

Update testCrossClusterRenameFullWildcard to expect only 3 user fields
instead of 9 (including 6 metadata fields), aligning the test with the
corrected behavior from PR opensearch-project#5350 where rename * no longer renames
hidden/metadata fields.

Signed-off-by: Songkan Tang <songkant@amazon.com>

---------

Signed-off-by: Songkan Tang <songkant@amazon.com>

Author: songkant-aws

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -670,6 +670,10 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) {
       }
 
       List<String> matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, newNames);
+      // Exclude metadata fields from wildcard rename (issue #5099)
+      if (WildcardRenameUtils.isWildcardPattern(sourcePattern)) {
+        matchingFields.removeIf(this::isMetadataField);
+      }
 
       for (String fieldName : matchingFields) {
         String newName =

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java

@@ -199,6 +199,41 @@ public void testRenameFullWildcard() throws IOException {
     verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20));
   }
 
+  @Test
+  public void testRenameFullWildcardExcludesMetadataFields() throws IOException {
+    JSONObject result =
+        executeQuery(String.format("source = %s | rename * as old_*", TEST_INDEX_STATE_COUNTRY));
+    verifySchema(
+        result,
+        schema("old_name", "string"),
+        schema("old_age", "int"),
+        schema("old_state", "string"),
+        schema("old_country", "string"),
+        schema("old_year", "int"),
+        schema("old_month", "int"));
+    verifyDataRows(
+        result,
+        rows("Jake", "USA", "California", 4, 2023, 70),
+        rows("Hello", "USA", "New York", 4, 2023, 30),
+        rows("John", "Canada", "Ontario", 4, 2023, 25),
+        rows("Jane", "Canada", "Quebec", 4, 2023, 20));
+  }
+
+  @Test
+  public void testRenamePartialWildcardExcludesMetadataFields() throws IOException {
+    JSONObject result =
+        executeQuery(String.format("source = %s | rename _* as meta_*", TEST_INDEX_STATE_COUNTRY));
+    verifySchema(
+        result,
+        schema("name", "string"),
+        schema("age", "int"),
+        schema("state", "string"),
+        schema("country", "string"),
+        schema("year", "int"),
+        schema("month", "int"));
+    verifyStandardDataRows(result);
+  }
+
   @Test
   public void testRenameMultipleWildcards() throws IOException {
     JSONObject result =

integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java

@@ -262,27 +262,12 @@ public void testCrossClusterRenameFullWildcard() throws IOException {
     JSONObject result =
         executeQuery(String.format("search source=%s | rename * as old_*", TEST_INDEX_DOG_REMOTE));
     verifyColumn(
-        result,
-        columnName("old_dog_name"),
-        columnName("old_holdersName"),
-        columnName("old_age"),
-        columnName("old__id"),
-        columnName("old__index"),
-        columnName("old__score"),
-        columnName("old__maxscore"),
-        columnName("old__sort"),
-        columnName("old__routing"));
+        result, columnName("old_dog_name"), columnName("old_holdersName"), columnName("old_age"));
     verifySchema(
         result,
         schema("old_dog_name", "string"),
         schema("old_holdersName", "string"),
-        schema("old_age", "bigint"),
-        schema("old__id", "string"),
-        schema("old__index", "string"),
-        schema("old__score", "float"),
-        schema("old__maxscore", "float"),
-        schema("old__sort", "bigint"),
-        schema("old__routing", "string"));
+        schema("old_age", "bigint"));
   }
 
   @Test

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5099.yml

@@ -0,0 +1,59 @@
+setup:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: true
+  - do:
+      indices.create:
+        index: issue5099
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              name:
+                type: keyword
+              age:
+                type: integer
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "issue5099", "_id": "1"}}'
+          - '{"name": "Alice", "age": 30}'
+          - '{"index": {"_index": "issue5099", "_id": "2"}}'
+          - '{"name": "Bob", "age": 25}'
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: issue5099
+        ignore_unavailable: true
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled: false
+
+---
+"Issue 5099: rename with wildcard should not apply on hidden fields":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      allowed_warnings:
+        - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled'
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=issue5099 | rename * as old_*
+
+  - match: { total: 2 }
+  - length: { schema: 2 }
+  - match: { schema: [ { name: "old_name", type: "string" }, { name: "old_age", type: "int" } ] }
+  - length: { datarows: 2 }