ArcadeData#3996 fix(cypher): CALL...YIELD preserves variables carried in from WITH (ArcadeData#4009)

Author: robfrank

engine/src/main/java/com/arcadedb/query/opencypher/executor/steps/CallStep.java

@@ -118,8 +118,12 @@ private ResultSet executeChainedCall(final CommandContext context, final int nRe
     final boolean hasYield = callClause.hasYield() && !callClause.isYieldAll();
     final boolean yieldHasWhere = hasYield && callClause.getYieldWhere() != null;
 
-    // Collect all results from all input rows
-    final List<Iterator<?>> allIters = new ArrayList<>();
+    // Each entry pairs the originating inputRow with the procedure's result iterator.
+    // Pairing is required so variables carried in from preceding WITH/MATCH clauses
+    // can be merged into every yielded result (issue #3996).
+    // Capacity is bounded by nRecords (the upstream batch limit); capped at 1M to guard
+    // against Integer.MAX_VALUE being passed as a "fetch all" sentinel.
+    final List<Map.Entry<Result, Iterator<?>>> allPairs = new ArrayList<>(nRecords > 0 && nRecords < 1000000 ? nRecords : 10);
     while (prevResults.hasNext()) {
       final Result inputRow = prevResults.next();
       final long begin = context.isProfiling() ? System.nanoTime() : 0;
@@ -131,17 +135,24 @@ private ResultSet executeChainedCall(final CommandContext context, final int nRe
 
         if (callResult == null) {
           if (callClause.isOptional())
-            allIters.add(java.util.Collections.singletonList((Object) mergeWithInputRow(inputRow, null)).iterator());
+            // Use an empty result so YIELD only sees the procedure's outputs (null for every
+            // field). Pre-merging inputRow here would let YIELD incorrectly read outer-scope
+            // variables if they share a name with a YIELD field. The lazy iterator merges
+            // inputRow later, after YIELD filtering.
+            allPairs.add(Map.entry(inputRow,
+                java.util.Collections.singletonList((Object) new ResultInternal()).iterator()));
           continue;
         }
 
+        final Iterator<?> iter;
         if (callResult instanceof Iterator) {
-          allIters.add((Iterator<?>) callResult);
+          iter = (Iterator<?>) callResult;
         } else if (callResult instanceof Collection) {
-          allIters.add(((Collection<?>) callResult).iterator());
+          iter = ((Collection<?>) callResult).iterator();
         } else {
-          allIters.add(java.util.Collections.singletonList(callResult).iterator());
+          iter = java.util.Collections.singletonList(callResult).iterator();
         }
+        allPairs.add(Map.entry(inputRow, iter));
       } finally {
         if (context.isProfiling())
           cost += (System.nanoTime() - begin);
@@ -176,9 +187,12 @@ public void close() {
       };
     }
 
-    // Standard path: lazily iterate through all result iterators
-    final Iterator<Iterator<?>> iterOfIters = allIters.iterator();
+    // Standard path: lazily iterate through all (inputRow, resultIterator) pairs.
+    // Each yielded result is merged with its originating inputRow so that variables
+    // from a preceding WITH/MATCH clause remain visible after CALL ... YIELD.
+    final Iterator<Map.Entry<Result, Iterator<?>>> pairIter = allPairs.iterator();
     final Iterator<Result> lazyIter = new Iterator<>() {
+      private Result currentInputRow = null;
       private Iterator<?> currentIter = null;
       private Result next = null;
 
@@ -190,15 +204,17 @@ public boolean hasNext() {
             if (hasYield) {
               final ResultInternal filtered = applyYieldToSingleResult(converted);
               if (filtered != null) {
-                next = filtered;
+                next = mergeWithInputRow(currentInputRow, filtered);
                 return true;
               }
             } else {
-              next = converted;
+              next = mergeWithInputRow(currentInputRow, converted);
               return true;
             }
-          } else if (iterOfIters.hasNext()) {
-            currentIter = iterOfIters.next();
+          } else if (pairIter.hasNext()) {
+            final Map.Entry<Result, Iterator<?>> pair = pairIter.next();
+            currentInputRow = pair.getKey();
+            currentIter = pair.getValue();
           } else {
             return false;
           }

engine/src/test/java/com/arcadedb/query/opencypher/CypherCallYieldWithVariablesTest.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.query.opencypher;
+
+import com.arcadedb.database.Database;
+import com.arcadedb.database.DatabaseFactory;
+import com.arcadedb.query.sql.executor.Result;
+import com.arcadedb.query.sql.executor.ResultSet;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Regression tests for GitHub issue #3996:
+ * CALL ... YIELD may null out variables carried in from WITH.
+ */
+class CypherCallYieldWithVariablesTest {
+  private Database database;
+
+  @BeforeEach
+  void setUp() {
+    database = new DatabaseFactory("./target/databases/testopencypher-call-yield-with").create();
+    final var personType = database.getSchema().createVertexType("Person");
+    personType.createProperty("name", String.class);
+    database.getSchema().createEdgeType("KNOWS");
+    database.transaction(() -> database.command("opencypher", "CREATE (:Person {name: 'Alice'})"));
+  }
+
+  @AfterEach
+  void tearDown() {
+    if (database != null) {
+      database.drop();
+      database = null;
+    }
+  }
+
+  @Test
+  void withLiteralPreservedAcrossCallDbLabels() {
+    final ResultSet rs = database.query("opencypher",
+        "WITH 1 AS x CALL db.labels() YIELD label RETURN x, label");
+
+    final List<Result> rows = new ArrayList<>();
+    while (rs.hasNext())
+      rows.add(rs.next());
+
+    assertThat(rows).isNotEmpty();
+    for (final Result row : rows) {
+      final Number x = row.getProperty("x");
+      assertThat(x).as("x must not be null across CALL db.labels()").isNotNull();
+      assertThat(x.longValue()).isEqualTo(1L);
+      assertThat((Object) row.getProperty("label")).as("label must not be null").isNotNull();
+    }
+  }
+
+  @Test
+  void withLiteralPreservedAcrossCallDbRelationshipTypes() {
+    final ResultSet rs = database.query("opencypher",
+        "WITH 1 AS x CALL db.relationshipTypes() YIELD relationshipType RETURN x, relationshipType");
+
+    final List<Result> rows = new ArrayList<>();
+    while (rs.hasNext())
+      rows.add(rs.next());
+
+    assertThat(rows).isNotEmpty();
+    for (final Result row : rows) {
+      final Number x = row.getProperty("x");
+      assertThat(x).as("x must not be null across CALL db.relationshipTypes()").isNotNull();
+      assertThat(x.longValue()).isEqualTo(1L);
+      assertThat((Object) row.getProperty("relationshipType")).as("relationshipType must not be null").isNotNull();
+    }
+  }
+
+  @Test
+  void withLiteralPreservedAcrossCallDbPropertyKeys() {
+    final ResultSet rs = database.query("opencypher",
+        "WITH 1 AS x CALL db.propertyKeys() YIELD propertyKey RETURN x, propertyKey");
+
+    final List<Result> rows = new ArrayList<>();
+    while (rs.hasNext())
+      rows.add(rs.next());
+
+    assertThat(rows).isNotEmpty();
+    for (final Result row : rows) {
+      final Number x = row.getProperty("x");
+      assertThat(x).as("x must not be null across CALL db.propertyKeys()").isNotNull();
+      assertThat(x.longValue()).isEqualTo(1L);
+      assertThat((Object) row.getProperty("propertyKey")).as("propertyKey must not be null").isNotNull();
+    }
+  }
+
+  @Test
+  void aggregatedValuePreservedAcrossCallDbLabels() {
+    // Aggregated values carried through WITH must also survive CALL ... YIELD
+    final ResultSet rs = database.query("opencypher",
+        "MATCH (:Person) WITH count(*) AS c CALL db.labels() YIELD label RETURN c, label");
+
+    final List<Result> rows = new ArrayList<>();
+    while (rs.hasNext())
+      rows.add(rs.next());
+
+    assertThat(rows).isNotEmpty();
+    for (final Result row : rows) {
+      final Number c = row.getProperty("c");
+      assertThat(c).as("count(*) must not be null across CALL db.labels()").isNotNull();
+      assertThat(c.longValue()).as("count must be 1").isEqualTo(1L);
+      assertThat((Object) row.getProperty("label")).as("label must not be null").isNotNull();
+    }
+  }
+}