add test cases for metadata and timestamp identifier

Signed-off-by: xinyual <xinyual@amazon.com>

Author: xinyual

core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java

@@ -5,6 +5,8 @@
 
 package org.opensearch.sql.ast.expression;
 
+import static org.opensearch.sql.utils.QueryStringUtils.maskField;
+
 import java.util.Arrays;
 import java.util.List;
 import lombok.EqualsAndHashCode;
@@ -65,7 +67,8 @@ public String toQueryString() {
 
   @Override
   public String toAnonymizedString() {
-    return "identifier " + operator.symbol + " ***";
+    String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString());
+    return maskField(fieldName) + " " + operator.symbol + " ***";
   }
 
   @Override

core/src/main/java/org/opensearch/sql/utils/QueryStringUtils.java

@@ -8,6 +8,26 @@
 /** Utility class for query_string syntax operations. */
 public class QueryStringUtils {
 
+  private static final String INTERNAL_TIMESTAMP = "@timestamp";
+
+  public static final String MASK_LITERAL = "***";
+
+  public static final String MASK_COLUMN = "identifier";
+
+  public static final String MASK_TIMESTAMP_COLUMN = "timestamp_identifier";
+
+  public static final String MASK_METADATA_COLUMN = "metadata_identifier";
+
+  public static String maskField(String fieldName) {
+    if (fieldName.equals(INTERNAL_TIMESTAMP)) {
+      return MASK_TIMESTAMP_COLUMN;
+    }
+    if (fieldName.startsWith("_")) {
+      return MASK_METADATA_COLUMN;
+    }
+    return MASK_COLUMN;
+  }
+
   // For field names, we typically don't escape dots as they're used for nested fields
   // But we escape other special characters
   public static final String LUCENE_SPECIAL_CHARS = "+-&|!(){}[]^\"~:/";

ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java

@@ -7,6 +7,9 @@
 
 import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation;
 import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild;
+import static org.opensearch.sql.utils.QueryStringUtils.MASK_COLUMN;
+import static org.opensearch.sql.utils.QueryStringUtils.MASK_LITERAL;
+import static org.opensearch.sql.utils.QueryStringUtils.maskField;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -108,10 +111,6 @@
 /** Utility class to mask sensitive information in incoming PPL queries. */
 public class PPLQueryDataAnonymizer extends AbstractNodeVisitor<String, String> {
 
-  public static final String MASK_LITERAL = "***";
-
-  public static final String MASK_COLUMN = "identifier";
-
   public static final String MASK_TABLE = "table";
 
   private final AnonymizerExpressionAnalyzer expressionAnalyzer;
@@ -916,7 +915,8 @@ public String visitIn(In node, String context) {
 
     @Override
     public String visitField(Field node, String context) {
-      return MASK_COLUMN;
+      String fieldName = node.getField().toString();
+      return maskField(fieldName);
     }
 
     @Override

ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java

@@ -255,8 +255,8 @@ public void testReverseCommand() {
   @Test
   public void testTimechartCommand() {
     assertEquals(
-        "source=table | timechart limit=*** useother=*** count() by span(identifier, *** m)"
-            + " identifier",
+        "source=table | timechart limit=*** useother=*** count() by span(timestamp_identifier, ***"
+            + " m) identifier",
         anonymize("source=t | timechart count() by host"));
   }
 
@@ -388,6 +388,13 @@ public void testAndExpression() {
         anonymize("source=t | where a=1 and b=2"));
   }
 
+  @Test
+  public void testAndExpressionWithMetaData() {
+    assertEquals(
+        "source=table | where metadata_identifier = *** and identifier = ***",
+        anonymize("source=t | where _id=1 and b=2"));
+  }
+
   @Test
   public void testOrExpression() {
     assertEquals(
@@ -879,7 +886,7 @@ private String anonymizeStatement(String query, boolean isExplain) {
   @Test
   public void testSearchWithAbsoluteTimeRange() {
     assertEquals(
-        "source=table (identifier >= ***) AND (identifier <= ***)",
+        "source=table (timestamp_identifier >= ***) AND (timestamp_identifier <= ***)",
         anonymize("search source=t earliest='2012-12-10 15:00:00' latest=now"));
   }
 
@@ -906,7 +913,7 @@ public void testSearchWithGroup() {
   @Test
   public void testSearchWithOr() {
     assertEquals(
-        "source=table (identifier >= *** OR identifier <= ***)",
+        "source=table (timestamp_identifier >= *** OR timestamp_identifier <= ***)",
         anonymize("search source=t earliest='2012-12-10 15:00:00' or latest=now"));
   }