Fix COALESCE(null, 42) returning wrong type string instead of int

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -499,8 +499,16 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) {
     List<RexNode> arguments = new ArrayList<>();
 
     boolean isCoalesce = "coalesce".equalsIgnoreCase(node.getFuncName());
+    // Save the previous state so nested non-COALESCE functions don't inherit the flag.
+    // This ensures that only direct arguments of COALESCE get the null-replacement
+    // behavior for unresolved field names, not deeply nested expressions.
+    boolean wasInCoalesce = context.isInCoalesceFunction();
     if (isCoalesce) {
       context.setInCoalesceFunction(true);
+    } else {
+      // Clear the flag for non-COALESCE nested functions so that unresolved fields
+      // inside e.g. array_compact(does_not_exist) still throw errors properly.
+      context.setInCoalesceFunction(false);
     }
 
     List<RexNode> capturedVars = null;
@@ -529,9 +537,8 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) {
         }
       }
     } finally {
-      if (isCoalesce) {
-        context.setInCoalesceFunction(false);
-      }
+      // Restore the previous inCoalesceFunction state
+      context.setInCoalesceFunction(wasInCoalesce);
     }
 
     // For transform/mvmap functions with captured variables, add them as additional arguments

core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java

@@ -315,9 +315,11 @@ private static Optional<RexNode> resolveLambdaVariable(
   private static Optional<RexNode> replaceWithNullLiteralInCoalesce(CalcitePlanContext context) {
     log.debug("replaceWithNullLiteralInCoalesce() called");
     if (context.isInCoalesceFunction()) {
+      // Use NULL type instead of VARCHAR so that COALESCE return type inference
+      // can derive the correct type from the non-null operands (fixes #5175).
       return Optional.of(
           context.rexBuilder.makeNullLiteral(
-              context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR)));
+              context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL)));
     }
     return Optional.empty();
   }

core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java

@@ -93,11 +93,26 @@ public SqlReturnTypeInference getReturnTypeInference() {
     return opBinding -> {
       var operandTypes = opBinding.collectOperandTypes();
 
-      // Let Calcite determine the least restrictive common type
-      var commonType = opBinding.getTypeFactory().leastRestrictive(operandTypes);
-      return commonType != null
-          ? commonType
-          : opBinding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
+      // Filter out NULL-typed operands so that leastRestrictive can find a common type
+      // among the concrete types.  Without this, COALESCE(null, 42) would fall back to
+      // VARCHAR because leastRestrictive([NULL, INTEGER]) returns null.
+      var nonNullTypes =
+          operandTypes.stream().filter(t -> t.getSqlTypeName() != SqlTypeName.NULL).toList();
+
+      if (nonNullTypes.isEmpty()) {
+        // All operands are NULL literals -- return nullable NULL type
+        return opBinding
+            .getTypeFactory()
+            .createTypeWithNullability(
+                opBinding.getTypeFactory().createSqlType(SqlTypeName.NULL), true);
+      }
+
+      var commonType = opBinding.getTypeFactory().leastRestrictive(nonNullTypes);
+      if (commonType != null) {
+        // Ensure the result is nullable since at least one operand could be NULL
+        return opBinding.getTypeFactory().createTypeWithNullability(commonType, true);
+      }
+      return opBinding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
     };
   }
 

ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEnhancedCoalesceTest.java

@@ -138,7 +138,7 @@ public void testCoalesceWithNonExistentField() {
     RelNode root = getRelNode(ppl);
     String expectedLogical =
         "LogicalSort(fetch=[2])\n"
-            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, $1)])\n"
+            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:NULL, $1)])\n"
             + "    LogicalTableScan(table=[[scott, EMP]])\n";
     verifyLogical(root, expectedLogical);
 
@@ -155,7 +155,7 @@ public void testCoalesceWithMultipleNonExistentFields() {
     RelNode root = getRelNode(ppl);
     String expectedLogical =
         "LogicalSort(fetch=[1])\n"
-            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, null:VARCHAR, $1,"
+            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:NULL, null:NULL, $1,"
             + " 'fallback':VARCHAR)])\n"
             + "    LogicalTableScan(table=[[scott, EMP]])\n";
     verifyLogical(root, expectedLogical);
@@ -175,8 +175,8 @@ public void testCoalesceWithAllNonExistentFields() {
     RelNode root = getRelNode(ppl);
     String expectedLogical =
         "LogicalSort(fetch=[1])\n"
-            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:VARCHAR, null:VARCHAR,"
-            + " null:VARCHAR)])\n"
+            + "  LogicalProject(EMPNO=[$0], result=[COALESCE(null:NULL, null:NULL,"
+            + " null:NULL)])\n"
             + "    LogicalTableScan(table=[[scott, EMP]])\n";
     verifyLogical(root, expectedLogical);
 
@@ -187,6 +187,54 @@ public void testCoalesceWithAllNonExistentFields() {
     verifyPPLToSparkSQL(root, expectedSparkSql);
   }
 
+  @Test
+  public void testCoalesceNullWithInteger() {
+    // Regression test for #5175: COALESCE(null, 42) should return INTEGER, not VARCHAR
+    String ppl = "source=EMP | eval x = coalesce(null, 42) | fields EMPNO, x | head 1";
+    RelNode root = getRelNode(ppl);
+    String expectedLogical =
+        "LogicalSort(fetch=[1])\n"
+            + "  LogicalProject(EMPNO=[$0], x=[COALESCE(null:NULL, 42)])\n"
+            + "    LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+    // Verify the COALESCE return type is INTEGER, not VARCHAR
+    org.apache.calcite.rel.logical.LogicalSort sort =
+        (org.apache.calcite.rel.logical.LogicalSort) root;
+    org.apache.calcite.rel.logical.LogicalProject proj =
+        (org.apache.calcite.rel.logical.LogicalProject) sort.getInput();
+    org.apache.calcite.rex.RexNode coalesceExpr = proj.getProjects().get(1);
+    org.junit.Assert.assertEquals(
+        org.apache.calcite.sql.type.SqlTypeName.INTEGER, coalesceExpr.getType().getSqlTypeName());
+  }
+
+  @Test
+  public void testCoalesceIntegerWithNull() {
+    // Regression test for #5175: COALESCE(42, null) should return INTEGER, not VARCHAR
+    String ppl = "source=EMP | eval x = coalesce(42, null) | fields EMPNO, x | head 1";
+    RelNode root = getRelNode(ppl);
+    String expectedLogical =
+        "LogicalSort(fetch=[1])\n"
+            + "  LogicalProject(EMPNO=[$0], x=[COALESCE(42, null:NULL)])\n"
+            + "    LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+    // Verify the COALESCE return type is INTEGER, not VARCHAR
+    org.apache.calcite.rel.logical.LogicalSort sort =
+        (org.apache.calcite.rel.logical.LogicalSort) root;
+    org.apache.calcite.rel.logical.LogicalProject proj =
+        (org.apache.calcite.rel.logical.LogicalProject) sort.getInput();
+    org.apache.calcite.rex.RexNode coalesceExpr = proj.getProjects().get(1);
+    org.junit.Assert.assertEquals(
+        org.apache.calcite.sql.type.SqlTypeName.INTEGER, coalesceExpr.getType().getSqlTypeName());
+  }
+
+  @Test
+  public void testCoalesceNullWithIntegerResult() {
+    // Regression test for #5175: COALESCE(null, 42) should return numeric 42, not string "42"
+    String ppl = "source=EMP | eval x = coalesce(null, 42) | fields x | head 1";
+    RelNode root = getRelNode(ppl);
+    verifyResult(root, "x=42\n");
+  }
+
   @Test
   public void testCoalesceWithEmptyString() {
     String ppl = "source=EMP | eval result = coalesce('', ENAME) | fields EMPNO, result | head 1";