Rewrite IS NOT NULL / IS NULL to native exists DSL in v2 filter pushdown

The v2 FilterQueryBuilder's luceneQueries map did not register entries for
BuiltinFunctionName.IS_NULL or IS_NOT_NULL, so visitFunction fell through to
buildScriptQuery for these unary predicates. This produced a compounded script
query in the pushed-down DSL, e.g.:

    {"script": {"script": {"source": "{\"langType\":\"v2\",\"script\":\"is not null(age)\"}", ...}}}

instead of the native OpenSearch exists query. Script fallback is more
expensive to evaluate, is not supported on AOSS / serverless, and diverges
from what the Calcite path and downstream tooling already produce.

This change adds a new ExistsQuery lucene query class that overrides the
LuceneQuery.canSupport(FunctionExpression) / build(FunctionExpression) pair
to accept a single ReferenceExpression argument (the standard base class
contract is binary {ref, literal}). Two instances are registered in
FilterQueryBuilder, one for each predicate sense, producing:

    IS NOT NULL col  ->  {"exists": {"field": "col"}}
    IS NULL col      ->  {"bool": {"must_not": [{"exists": {"field": "col"}}]}}

matching the Calcite PredicateAnalyzer output for these predicates.

Nested-field predicates are explicitly guarded: ExistsQuery.canSupport
returns false when the argument is a nested() function, mirroring the
equivalent guard in PredicateAnalyzer ("OpenSearch DSL does not handle
IS_NULL / IS_NOT_NULL on nested fields correctly"). When canSupport returns
false the existing default branch in FilterQueryBuilder.visitFunction falls
through to the script query path, preserving the prior behavior for that
edge case.

This closes the tracker row "Pushdown coverage - IS NOT NULL pushes script
instead of exists". The change affects all v2 SQL queries, not just
vectorSearch table function queries.

Test coverage:
- FilterQueryBuilderTest: updated should_build_script_query_for_unsupported_lucene_query
  to two new assertions - should_build_exists_query_for_is_not_null and
  should_build_must_not_exists_query_for_is_null - pinning the new DSL shape.
- ExistsPushdownIT (new): explain-plan integration tests asserting that
  WHERE col IS NOT NULL pushes as native exists DSL (no "script" key) and
  WHERE col IS NULL pushes as bool/must_not[exists].
- AggregationIT (existing) continues to pass, confirming result-level
  correctness is preserved for queries like "where int0 IS NOT NULL".

Signed-off-by: Eric Wei <mengwei.eric@gmail.com>

Author: mengweieric

integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.sql;
+
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.junit.Test;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+import org.opensearch.sql.legacy.TestsConstants;
+
+/**
+ * Explain-plan integration tests asserting that {@code IS NOT NULL} / {@code IS NULL} predicates
+ * push down as native OpenSearch {@code exists} DSL rather than as serialized script queries.
+ *
+ * <p>Before this change both predicates serialized through the compounded script engine, producing
+ * a {@code "script"} clause in the pushdown DSL. After this change the v2 filter builder emits
+ * {@code {"exists": {"field": ...}}} directly for {@code IS NOT NULL}, and a {@code bool} query
+ * with a single {@code must_not[exists]} child for {@code IS NULL}. This matches what downstream
+ * tooling, serverless / AOSS, and the Calcite path already produce.
+ */
+public class ExistsPushdownIT extends SQLIntegTestCase {
+
+  // Anchored on the surrounding `sourceBuilder=...`, `pitId=` tokens in OpenSearchRequest's
+  // toString() output. Test-only coupling: if that request-string format changes (token renamed,
+  // pitId removed), this helper breaks even when the DSL shape is still correct. Update the regex
+  // anchors if that happens.
+  private static final Pattern SOURCE_BUILDER_JSON =
+      Pattern.compile("sourceBuilder=(\\{.*?\\}), pitId=", Pattern.DOTALL);
+
+  /** Extracts and unescapes the sourceBuilder JSON embedded in the explain request string. */
+  private static String extractSourceBuilderJson(String explain) {
+    Matcher m = SOURCE_BUILDER_JSON.matcher(explain);
+    assertTrue("Explain should contain sourceBuilder JSON:\n" + explain, m.find());
+    return m.group(1).replace("\\\"", "\"");
+  }
+
+  @Override
+  protected void init() throws Exception {
+    loadIndex(Index.ACCOUNT);
+  }
+
+  private static final String TEST_INDEX = TestsConstants.TEST_INDEX_ACCOUNT;
+
+  @Test
+  public void testIsNotNullPushesDownAsExistsQuery() throws IOException {
+    String explain =
+        explainQuery("SELECT age FROM " + TEST_INDEX + " WHERE age IS NOT NULL LIMIT 1");
+    String sourceBuilder = extractSourceBuilderJson(explain);
+
+    assertTrue(
+        "IS NOT NULL should push down as native exists DSL:\n" + sourceBuilder,
+        sourceBuilder.contains("\"exists\""));
+    assertTrue(
+        "IS NOT NULL exists DSL should target the 'age' field:\n" + sourceBuilder,
+        sourceBuilder.contains("\"field\":\"age\""));
+    assertFalse(
+        "IS NOT NULL should not fall through to a script query:\n" + sourceBuilder,
+        sourceBuilder.contains("\"script\""));
+  }
+
+  @Test
+  public void testIsNullPushesDownAsMustNotExistsQuery() throws IOException {
+    String explain = explainQuery("SELECT age FROM " + TEST_INDEX + " WHERE age IS NULL LIMIT 1");
+    String sourceBuilder = extractSourceBuilderJson(explain);
+
+    assertTrue(
+        "IS NULL should push down as bool/must_not[exists] DSL:\n" + sourceBuilder,
+        sourceBuilder.contains("\"must_not\""));
+    assertTrue(
+        "IS NULL should wrap a native exists clause:\n" + sourceBuilder,
+        sourceBuilder.contains("\"exists\""));
+    assertTrue(
+        "IS NULL exists DSL should target the 'age' field:\n" + sourceBuilder,
+        sourceBuilder.contains("\"field\":\"age\""));
+    assertFalse(
+        "IS NULL should not fall through to a script query:\n" + sourceBuilder,
+        sourceBuilder.contains("\"script\""));
+  }
+}

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java

@@ -29,6 +29,7 @@
 import org.opensearch.sql.expression.function.FunctionName;
 import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine.ScriptEngineType;
 import org.opensearch.sql.opensearch.storage.script.core.ExpressionScript;
+import org.opensearch.sql.opensearch.storage.script.filter.lucene.ExistsQuery;
 import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery;
 import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery;
 import org.opensearch.sql.opensearch.storage.script.filter.lucene.NestedQuery;
@@ -86,6 +87,8 @@ public ScriptQueryUnSupportedException(String message) {
           .put(BuiltinFunctionName.WILDCARD_QUERY.getName(), new WildcardQuery())
           .put(BuiltinFunctionName.WILDCARDQUERY.getName(), new WildcardQuery())
           .put(BuiltinFunctionName.NESTED.getName(), new NestedQuery())
+          .put(BuiltinFunctionName.IS_NULL.getName(), new ExistsQuery(true /* negated */))
+          .put(BuiltinFunctionName.IS_NOT_NULL.getName(), new ExistsQuery(false))
           .build();
 
   /**

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/ExistsQuery.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.opensearch.storage.script.filter.lucene;
+
+import static org.opensearch.sql.analysis.NestedAnalyzer.isNestedFunction;
+
+import lombok.RequiredArgsConstructor;
+import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.sql.expression.FunctionExpression;
+import org.opensearch.sql.expression.ReferenceExpression;
+
+/**
+ * Lucene query that builds a native {@code exists} DSL fragment for {@code IS NULL} / {@code IS NOT
+ * NULL} predicates.
+ *
+ * <p>This replaces the previous behavior of serializing these unary predicates as compounded script
+ * queries. The native {@code exists} query is cheaper, AOSS / serverless compatible, and the
+ * expected DSL shape downstream consumers look for.
+ *
+ * <p>Unlike most {@link LuceneQuery} subclasses this predicate family is unary (a single reference
+ * argument) rather than the standard {ref, literal} pair, so this class overrides both {@link
+ * #canSupport(FunctionExpression)} and {@link #build(FunctionExpression)}.
+ *
+ * <p>Nested-field predicates are intentionally NOT supported here: OpenSearch DSL does not handle
+ * {@code IS_NULL} / {@code IS_NOT_NULL} on nested fields correctly (see the equivalent guard in
+ * {@code PredicateAnalyzer} for the Calcite path). When the reference is a nested function, {@link
+ * #canSupport} returns {@code false} and {@link
+ * org.opensearch.sql.opensearch.storage.script.filter.FilterQueryBuilder} falls back to the script
+ * query path, preserving correctness.
+ */
+@RequiredArgsConstructor
+public class ExistsQuery extends LuceneQuery {
+
+  /** When true, the predicate is {@code IS NULL} and the exists query is wrapped in must_not. */
+  private final boolean negated;
+
+  @Override
+  public boolean canSupport(FunctionExpression func) {
+    return func.getArguments().size() == 1
+        && func.getArguments().get(0) instanceof ReferenceExpression
+        && !isNestedFunction(func.getArguments().get(0));
+  }
+
+  @Override
+  public QueryBuilder build(FunctionExpression func) {
+    ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0);
+    String fieldName = ref.getRawPath();
+    QueryBuilder existsQuery = QueryBuilders.existsQuery(fieldName);
+    if (negated) {
+      return QueryBuilders.boolQuery().mustNot(existsQuery);
+    }
+    return existsQuery;
+  }
+}

opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java

@@ -174,22 +174,37 @@ void should_build_wildcard_query_for_like_expression() {
   }
 
   @Test
-  void should_build_script_query_for_unsupported_lucene_query() {
-    mockToStringSerializer();
+  void should_build_exists_query_for_is_not_null() {
     assertJsonEquals(
         "{\n"
-            + "  \"script\" : {\n"
-            + "    \"script\" : {\n"
-            + "      \"source\" : \"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"is not"
-            + " null(age)\\\"}\",\n"
-            + "      \"lang\" : \"opensearch_compounded_script\"\n"
-            + "    },\n"
+            + "  \"exists\" : {\n"
+            + "    \"field\" : \"age\",\n"
             + "    \"boost\" : 1.0\n"
             + "  }\n"
             + "}",
         buildQuery(DSL.isnotnull(ref("age", INTEGER))));
   }
 
+  @Test
+  void should_build_must_not_exists_query_for_is_null() {
+    assertJsonEquals(
+        "{\n"
+            + "  \"bool\" : {\n"
+            + "    \"must_not\" : [\n"
+            + "      {\n"
+            + "        \"exists\" : {\n"
+            + "          \"field\" : \"age\",\n"
+            + "          \"boost\" : 1.0\n"
+            + "        }\n"
+            + "      }\n"
+            + "    ],\n"
+            + "    \"adjust_pure_negative\" : true,\n"
+            + "    \"boost\" : 1.0\n"
+            + "  }\n"
+            + "}",
+        buildQuery(DSL.is_null(ref("age", INTEGER))));
+  }
+
   @Test
   void should_build_script_query_for_function_expression() {
     mockToStringSerializer();