Add max match limit implementation

Signed-off-by: Jialiang Liang <jiallian@amazon.com>

Author: RyanL1997

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -29,6 +29,7 @@ public enum Key {
     PATTERN_MODE("plugins.ppl.pattern.mode"),
     PATTERN_MAX_SAMPLE_COUNT("plugins.ppl.pattern.max.sample.count"),
     PATTERN_BUFFER_LIMIT("plugins.ppl.pattern.buffer.limit"),
+    PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"),
 
     /** Enable Calcite as execution engine */
     CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),

docs/user/ppl/cmd/rex.rst

@@ -23,7 +23,7 @@ rex field=<field> <pattern> [max_match=<int>]
 
 * field: mandatory. The field must be a string field to extract data from.
 * pattern: mandatory string. The regular expression pattern with named capture groups used to extract new fields. Pattern must contain at least one named capture group using ``(?<name>pattern)`` syntax.
-* max_match: optional integer (default=1). Maximum number of matches to extract. If greater than 1, extracted fields become arrays.
+* max_match: optional integer (default=1). Maximum number of matches to extract. If greater than 1, extracted fields become arrays. The value 0 means unlimited matches, but is automatically capped to the configured limit (default: 10, configurable via ``plugins.ppl.rex.max_match.limit``).
 
 Example 1: Basic Field Extraction
 ==================================
@@ -135,6 +135,28 @@ Correct PPL query without underscores::
     +-----------------------+------------+-------------+
 
 
+Example 7: Max Match Limit Protection
+======================================
+
+Demonstrates the max_match limit protection mechanism. When max_match=0 (unlimited) is specified, the system automatically caps it to prevent memory exhaustion.
+
+PPL query with max_match=0 automatically capped to default limit of 10::
+
+    os> source=accounts | rex field=address "(?<digit>\\d*)" max_match=0 | eval digit_count=array_length(digit) | fields address, digit_count | head 1 ;
+    fetched rows / total rows = 1/1
+    +-----------------+-------------+
+    | address         | digit_count |
+    |-----------------+-------------|
+    | 880 Holmes Lane | 10          |
+    +-----------------+-------------+
+
+PPL query exceeding the configured limit results in an error::
+
+    os> source=accounts | rex field=address "(?<digit>\\d*)" max_match=100 | fields address, digit | head 1 ;
+    {'reason': 'Invalid Query', 'details': 'Rex command max_match value (100) exceeds the configured limit (10). Consider using a smaller max_match value or adjust the plugins.ppl.rex.max_match.limit setting.', 'type': 'IllegalArgumentException'}
+    Error: Query returned no data
+
+
 Comparison with Related Commands
 ================================
 
@@ -164,3 +186,10 @@ There are several important limitations with the rex command:
 
 - Pattern must contain at least one named capture group
 - Regular capture groups ``(...)`` without names are not allowed
+
+**Max Match Limit:**
+
+- The ``max_match`` parameter is subject to a configurable system limit to prevent memory exhaustion
+- When ``max_match=0`` (unlimited) is specified, it is automatically capped at the configured limit (default: 10)
+- User-specified values exceeding the configured limit will result in an error
+- Users can adjust the limit via the ``plugins.ppl.rex.max_match.limit`` cluster setting. Setting this limit to a large value is not recommended as it can lead to excessive memory consumption, especially with patterns that match empty strings (e.g., ``\d*``, ``\w*``)

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRexCommandIT.java

@@ -10,6 +10,7 @@
 import java.io.IOException;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
+import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
 
 public class CalciteRexCommandIT extends PPLIntegTestCase {
@@ -147,4 +148,98 @@ public void testRexWithStatsCommand() throws IOException {
     assertFalse(domain.contains("@"));
     assertTrue(domain.matches("[a-z]+"));
   }
+
+  @Test
+  public void testRexMaxMatchZeroLimitedToDefaultTen() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d*)\\\" max_match=0 | eval"
+                    + " digit_count=array_length(digit) | fields address, digit_count | head 1",
+                TEST_INDEX_ACCOUNT));
+
+    assertEquals(1, result.getJSONArray("datarows").length());
+    // Should be capped at 10 matches
+    assertEquals(10, result.getJSONArray("datarows").getJSONArray(0).get(1));
+  }
+
+  @Test
+  public void testRexMaxMatchExceedsDefaultLimit() throws IOException {
+    try {
+      executeQuery(
+          String.format(
+              "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d+)\\\" max_match=100 | fields"
+                  + " address, digit",
+              TEST_INDEX_ACCOUNT));
+      fail("Should have thrown an exception for max_match exceeding default limit");
+    } catch (Exception e) {
+      assertTrue(e.getMessage().contains("exceeds the configured limit (10)"));
+      assertTrue(e.getMessage().contains("Consider using a smaller max_match value"));
+    }
+  }
+
+  @Test
+  public void testRexMaxMatchWithinDefaultLimit() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d*)\\\" max_match=5 | eval"
+                    + " digit_count=array_length(digit) | fields address, digit_count | head 1",
+                TEST_INDEX_ACCOUNT));
+
+    assertEquals(1, result.getJSONArray("datarows").length());
+    // Should respect the specified limit of 5
+    assertEquals(5, result.getJSONArray("datarows").getJSONArray(0).get(1));
+  }
+
+  @Test
+  public void testRexMaxMatchAtDefaultLimit() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d*)\\\" max_match=10 | eval"
+                    + " digit_count=array_length(digit) | fields address, digit_count | head 1",
+                TEST_INDEX_ACCOUNT));
+
+    assertEquals(1, result.getJSONArray("datarows").length());
+    // Should accept exactly the limit
+    assertEquals(10, result.getJSONArray("datarows").getJSONArray(0).get(1));
+  }
+
+  @Test
+  public void testRexMaxMatchConfigurableLimit() throws IOException {
+    // Set a custom limit of 5
+    updateClusterSettings(
+        new ClusterSetting(PERSISTENT, Settings.Key.PPL_REX_MAX_MATCH_LIMIT.getKeyValue(), "5"));
+
+    try {
+      // Test that max_match=0 is capped to the new limit
+      JSONObject result =
+          executeQuery(
+              String.format(
+                  "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d*)\\\" max_match=0 | eval"
+                      + " digit_count=array_length(digit) | fields address, digit_count | head 1",
+                  TEST_INDEX_ACCOUNT));
+
+      assertEquals(1, result.getJSONArray("datarows").length());
+      // Should be capped at the configured limit of 5
+      assertEquals(5, result.getJSONArray("datarows").getJSONArray(0).get(1));
+
+      // Test that exceeding the custom limit throws an error
+      try {
+        executeQuery(
+            String.format(
+                "source=%s | rex field=address \\\"(?<digit>\\\\\\\\d+)\\\" max_match=10 | fields"
+                    + " address, digit",
+                TEST_INDEX_ACCOUNT));
+        fail("Should have thrown an exception for max_match exceeding custom limit");
+      } catch (Exception e) {
+        assertTrue(e.getMessage().contains("exceeds the configured limit (5)"));
+        assertTrue(e.getMessage().contains("adjust the plugins.ppl.rex.max_match.limit setting"));
+      }
+    } finally {
+      updateClusterSettings(
+          new ClusterSetting(PERSISTENT, Settings.Key.PPL_REX_MAX_MATCH_LIMIT.getKeyValue(), null));
+    }
+  }
 }

opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java

@@ -99,6 +99,14 @@ public class OpenSearchSettings extends Settings {
           Setting.Property.NodeScope,
           Setting.Property.Dynamic);
 
+  public static final Setting<?> PPL_REX_MAX_MATCH_LIMIT_SETTING =
+      Setting.intSetting(
+          Key.PPL_REX_MAX_MATCH_LIMIT.getKeyValue(),
+          10,
+          1,
+          Setting.Property.NodeScope,
+          Setting.Property.Dynamic);
+
   public static final Setting<?> CALCITE_ENGINE_ENABLED_SETTING =
       Setting.boolSetting(
           Key.CALCITE_ENGINE_ENABLED.getKeyValue(),
@@ -327,6 +335,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) {
         Key.PATTERN_BUFFER_LIMIT,
         DEFAULT_PATTERN_BUFFER_LIMIT_SETTING,
         new Updater(Key.PATTERN_BUFFER_LIMIT));
+    register(
+        settingBuilder,
+        clusterSettings,
+        Key.PPL_REX_MAX_MATCH_LIMIT,
+        PPL_REX_MAX_MATCH_LIMIT_SETTING,
+        new Updater(Key.PPL_REX_MAX_MATCH_LIMIT));
     register(
         settingBuilder,
         clusterSettings,
@@ -531,6 +545,7 @@ public static List<Setting<?>> pluginSettings() {
         .add(DEFAULT_PATTERN_MODE_SETTING)
         .add(DEFAULT_PATTERN_MAX_SAMPLE_COUNT_SETTING)
         .add(DEFAULT_PATTERN_BUFFER_LIMIT_SETTING)
+        .add(PPL_REX_MAX_MATCH_LIMIT_SETTING)
         .add(QUERY_MEMORY_LIMIT_SETTING)
         .add(QUERY_SIZE_LIMIT_SETTING)
         .add(METRICS_ROLLING_WINDOW_SETTING)

ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java

@@ -982,7 +982,31 @@ public UnresolvedPlan visitRexCommand(OpenSearchPPLParser.RexCommandContext ctx)
       }
     }
 
-    return new Rex(field, pattern, mode, maxMatch);
+    int maxMatchLimit =
+        (settings != null) ? settings.getSettingValue(Settings.Key.PPL_REX_MAX_MATCH_LIMIT) : 10;
+
+    int effectiveMaxMatch;
+    if (maxMatch.isPresent()) {
+      if (maxMatch.get() == 0) {
+        effectiveMaxMatch = maxMatchLimit;
+      } else if (maxMatch.get() > maxMatchLimit) {
+        throw new IllegalArgumentException(
+            String.format(
+                "Rex command max_match value (%d) exceeds the configured limit (%d). "
+                    + "Consider using a smaller max_match value"
+                    + (settings != null
+                        ? " or adjust the plugins.ppl.rex.max_match.limit setting."
+                        : "."),
+                maxMatch.get(),
+                maxMatchLimit));
+      } else {
+        effectiveMaxMatch = maxMatch.get();
+      }
+    } else {
+      effectiveMaxMatch = 1;
+    }
+
+    return new Rex(field, pattern, mode, Optional.of(effectiveMaxMatch));
   }
 
   /** Get original text in query. */

ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java

@@ -5,15 +5,24 @@
 
 package org.opensearch.sql.ppl.calcite;
 
+import static org.mockito.Mockito.doReturn;
+
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.test.CalciteAssert;
+import org.junit.Before;
 import org.junit.Test;
+import org.opensearch.sql.common.setting.Settings;
 
 public class CalcitePPLRexTest extends CalcitePPLAbstractTest {
   public CalcitePPLRexTest() {
     super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL);
   }
 
+  @Before
+  public void setUp() {
+    doReturn(10).when(settings).getSettingValue(Settings.Key.PPL_REX_MAX_MATCH_LIMIT);
+  }
+
   @Test
   public void testRexBasicFieldExtraction() {
     String ppl = "source=EMP | rex field=ENAME '(?<first>[A-Z]).*' | fields ENAME, first";
@@ -159,4 +168,62 @@ public void testRexWithSort() {
             + "LIMIT 5";
     verifyPPLToSparkSQL(root, expectedSparkSql);
   }
+
+  @Test
+  public void testRexWithMaxMatchZero() {
+    // Test that max_match=0 (unlimited) is capped to the configured limit
+    String ppl =
+        "source=EMP | rex field=ENAME '(?<letter>[A-Z])' max_match=0 | fields ENAME, letter";
+    RelNode root = getRelNode(ppl);
+    String expectedLogical =
+        "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?<letter>[A-Z])', 1, 10)])\n"
+            + "  LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+
+    String expectedSparkSql =
+        "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?<letter>[A-Z])', 1, 10) `letter`\n"
+            + "FROM `scott`.`EMP`";
+    verifyPPLToSparkSQL(root, expectedSparkSql);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testRexWithMaxMatchExceedsLimit() {
+    // Test that max_match exceeding the configured limit throws an exception
+    String ppl =
+        "source=EMP | rex field=ENAME '(?<letter>[A-Z])' max_match=100 | fields ENAME, letter";
+    getRelNode(ppl);
+  }
+
+  @Test
+  public void testRexWithMaxMatchWithinLimit() {
+    String ppl =
+        "source=EMP | rex field=ENAME '(?<letter>[A-Z])' max_match=5 | fields ENAME, letter";
+    RelNode root = getRelNode(ppl);
+    String expectedLogical =
+        "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?<letter>[A-Z])', 1, 5)])\n"
+            + "  LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+
+    String expectedSparkSql =
+        "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?<letter>[A-Z])', 1, 5) `letter`\n"
+            + "FROM `scott`.`EMP`";
+    verifyPPLToSparkSQL(root, expectedSparkSql);
+  }
+
+  @Test
+  public void testRexWithMaxMatchAtLimit() {
+    // Test that max_match exactly at the limit works
+    String ppl =
+        "source=EMP | rex field=ENAME '(?<letter>[A-Z])' max_match=10 | fields ENAME, letter";
+    RelNode root = getRelNode(ppl);
+    String expectedLogical =
+        "LogicalProject(ENAME=[$1], letter=[REX_EXTRACT_MULTI($1, '(?<letter>[A-Z])', 1, 10)])\n"
+            + "  LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+
+    String expectedSparkSql =
+        "SELECT `ENAME`, `REX_EXTRACT_MULTI`(`ENAME`, '(?<letter>[A-Z])', 1, 10) `letter`\n"
+            + "FROM `scott`.`EMP`";
+    verifyPPLToSparkSQL(root, expectedSparkSql);
+  }
 }