fix(retrieve): keep memory vs database retrieves distinct (#726)

A retrieve-by-association (in-memory) and a database retrieve with an
equivalent reverse-association XPath were indistinguishable in MDL, and
re-running a script turned the memory retrieve into a database retrieve —
silently changing behavior and breaking round-trip.

Two root causes, both from conflating the two forms:

1. Build (cmd_microflows_builder_actions.go): a reverse-Reference
   `from $var/Assoc` was rewritten to a DatabaseRetrieveSource, on the
   assumption that "an AssociationRetrieveSource always returns a single
   object." That's false — Mendix returns a *list* for reverse traversal and
   mxbuild accepts the AssociationRetrieveSource (verified: a reverse
   association retrieve feeding count() passes mx check with 0 errors). Now
   `from $var/Assoc` is ALWAYS an AssociationRetrieveSource; only the output
   variable's type varies by direction (single vs list).

2. Describe (cmd_microflows_format_action.go): a DatabaseRetrieveSource whose
   XPath was a reverse-association constraint was rendered with the association
   shorthand `from $var/Assoc`. Now a database retrieve ALWAYS renders as
   `from Entity where …`.

Verified end-to-end on 11.12: the two retrieves render distinctly, keep their
source types (AssociationRetrieveSource vs DatabaseRetrieveSource), round-trip
identically, and mx check reports 0 errors. Updated the reverse-reference
builder/format tests that encoded the old behavior.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ako

mdl-examples/bug-tests/726-retrieve-db-vs-memory-distinct.mdl

@@ -0,0 +1,50 @@
+-- ============================================================================
+-- Bug #726 — database and memory (association) retrieves indistinguishable
+-- ============================================================================
+--
+-- Symptom (before fix):
+--   A retrieve-by-association (in-memory) and a database retrieve whose XPath is
+--   a reverse-association constraint both rendered as the association shorthand
+--   `retrieve $x from $Order/Module.Assoc`, so they were indistinguishable. And
+--   re-running the script turned BOTH into database retrieves — the memory
+--   retrieve was silently lost.
+--
+--   Two root causes:
+--     1. Build: a reverse-Reference `from $var/Assoc` was rewritten to a
+--        DatabaseRetrieveSource (the heuristic wrongly assumed an
+--        AssociationRetrieveSource always returns a single object). Mendix in
+--        fact returns a list for reverse traversal and mxbuild accepts it.
+--     2. Describe: a DatabaseRetrieveSource with a reverse-association XPath was
+--        rendered using the association shorthand.
+--
+-- After fix:
+--   `from $var/Assoc` is ALWAYS an AssociationRetrieveSource (memory); a
+--   database retrieve ALWAYS renders as `from Entity where …`. The two are
+--   distinct and round-trip stably.
+--
+-- Usage:
+--   mxcli exec mdl-examples/bug-tests/726-retrieve-db-vs-memory-distinct.mdl -p app.mpr
+--   mxcli -p app.mpr -c "describe microflow TestBug726.ACT_get_test"
+--   The two retrieves must render differently (one `from $Order/…`, one
+--   `from …OrderLine where …`), and mx check must report 0 errors.
+-- ============================================================================
+
+create module TestBug726;
+
+create persistent entity TestBug726.Order ( OrderNo: string(50) );
+create persistent entity TestBug726.OrderLine ( LineNo: integer );
+create association TestBug726.OrderLine_Order
+  from TestBug726.OrderLine to TestBug726.Order
+  type reference owner default;
+/
+
+create microflow TestBug726.ACT_get_test ( $Order: TestBug726.Order )
+begin
+  -- memory / retrieve-by-association (reverse Reference -> list of OrderLine)
+  retrieve $OrderLineList_M from $Order/TestBug726.OrderLine_Order;
+  -- database retrieve with the equivalent XPath
+  retrieve $OrderLineList_DB from TestBug726.OrderLine
+    where TestBug726.OrderLine_Order = $Order;
+  return;
+end;
+/

mdl/executor/cmd_microflows_builder_actions.go

@@ -820,76 +820,59 @@ func (fb *flowBuilder) addRetrieveAction(s *ast.RetrieveStmt) model.ID {
 			startVarType = fb.varTypes[s.StartVariable]
 		}
 
-		outputUsedAsList := fb.listInputVariables != nil && fb.listInputVariables[s.Variable]
 		outputUsedAsObject := fb.objectInputVariables != nil && fb.objectInputVariables[s.Variable]
 		// startsFromChildSide is true when the retrieve's start variable is the
 		// child side of the association (or a subclass of it). Inheritance has
 		// to be honoured so traversals like `$httpRequest/System.HttpHeaders`
 		// — where HttpRequest extends HttpMessage and HttpHeaders has child
-		// HttpMessage — are still classified as reverse traversal.
+		// HttpMessage — are still classified as reverse traversal (which returns
+		// a list, so the output variable is typed accordingly below).
 		startsFromChildSide := assocInfo != nil &&
 			assocInfo.childEntityQN != "" &&
 			fb.entityIsSubtypeOf(startVarType, assocInfo.childEntityQN)
-		// Owner-both Reference associations need later usage context: the same
-		// compact retrieve can be consumed as either a list or a single object.
-		// Owner="" means metadata was unavailable, so keep the association source.
-		expandReverseReference := assocInfo != nil &&
-			assocInfo.Type == domainmodel.AssociationTypeReference &&
-			assocInfo.Owner != "" &&
-			assocInfo.parentPersistable &&
-			assocInfo.childEntityQN != "" &&
-			startsFromChildSide &&
-			(assocInfo.Owner != domainmodel.AssociationOwnerBoth || (outputUsedAsList && !outputUsedAsObject))
-
-		if expandReverseReference {
-			// Reverse traversal on Reference: child → parent (one-to-many)
-			// Use DatabaseRetrieveSource with XPath to get a list of parent entities
-			dbSource := &microflows.DatabaseRetrieveSource{
-				BaseElement:         model.BaseElement{ID: model.ID(types.GenerateID())},
-				EntityQualifiedName: assocInfo.parentEntityQN,
-				XPathConstraint:     "[" + assocQN + " = $" + s.StartVariable + "]",
-			}
-			source = dbSource
-			if fb.varTypes != nil {
-				fb.varTypes[s.Variable] = "List of " + assocInfo.parentEntityQN
-			}
-		} else {
-			// Forward traversal or ReferenceSet: use AssociationRetrieveSource
-			source = &microflows.AssociationRetrieveSource{
-				BaseElement:              model.BaseElement{ID: model.ID(types.GenerateID())},
-				StartVariable:            s.StartVariable,
-				AssociationQualifiedName: assocQN,
-			}
-			if fb.varTypes != nil {
-				if assocInfo != nil && assocInfo.Type == domainmodel.AssociationTypeReference {
-					// Forward Reference traversal returns a single object. Legacy or
-					// non-persistable reverse traversal can still use association
-					// source syntax, but keeps list typing for downstream actions.
-					otherEntity := assocInfo.childEntityQN
-					if startsFromChildSide {
-						otherEntity = assocInfo.parentEntityQN
-					}
-					if startsFromChildSide && !outputUsedAsObject {
-						fb.varTypes[s.Variable] = "List of " + otherEntity
-					} else {
-						fb.varTypes[s.Variable] = otherEntity
-					}
-				} else if assocInfo != nil && assocInfo.Type == domainmodel.AssociationTypeReferenceSet {
-					// ReferenceSet traversal returns a list of the entity on the other side,
-					// not a list typed as the association itself.
-					otherEntity := assocInfo.childEntityQN
-					if startsFromChildSide {
-						otherEntity = assocInfo.parentEntityQN
-					}
-					if otherEntity != "" {
-						fb.varTypes[s.Variable] = "List of " + otherEntity
-					} else {
-						fb.varTypes[s.Variable] = "List of " + assocQN
-					}
+
+		// The `$var/Module.Assoc` syntax is ALWAYS a retrieve-by-association (an
+		// in-memory retrieve), regardless of traversal direction. Mendix resolves
+		// the result multiplicity from the association metadata: a reverse
+		// Reference traversal (child → parent) returns a list, and mxbuild
+		// accepts an AssociationRetrieveSource for it. Rewriting it to a
+		// DatabaseRetrieveSource (as an earlier heuristic did) turns a memory
+		// retrieve into a database retrieve and makes the two indistinguishable
+		// on round-trip — issue #726. Only the output variable's *type* differs
+		// by direction; the source stays an AssociationRetrieveSource.
+		source = &microflows.AssociationRetrieveSource{
+			BaseElement:              model.BaseElement{ID: model.ID(types.GenerateID())},
+			StartVariable:            s.StartVariable,
+			AssociationQualifiedName: assocQN,
+		}
+		if fb.varTypes != nil {
+			if assocInfo != nil && assocInfo.Type == domainmodel.AssociationTypeReference {
+				// Reference: forward traversal (parent side) → single object;
+				// reverse traversal (child side) → list of the other entity.
+				otherEntity := assocInfo.childEntityQN
+				if startsFromChildSide {
+					otherEntity = assocInfo.parentEntityQN
+				}
+				if startsFromChildSide && !outputUsedAsObject {
+					fb.varTypes[s.Variable] = "List of " + otherEntity
+				} else {
+					fb.varTypes[s.Variable] = otherEntity
+				}
+			} else if assocInfo != nil && assocInfo.Type == domainmodel.AssociationTypeReferenceSet {
+				// ReferenceSet traversal returns a list of the entity on the other side,
+				// not a list typed as the association itself.
+				otherEntity := assocInfo.childEntityQN
+				if startsFromChildSide {
+					otherEntity = assocInfo.parentEntityQN
+				}
+				if otherEntity != "" {
+					fb.varTypes[s.Variable] = "List of " + otherEntity
 				} else {
-					// ReferenceSet or unknown: returns a list
 					fb.varTypes[s.Variable] = "List of " + assocQN
 				}
+			} else {
+				// ReferenceSet or unknown: returns a list
+				fb.varTypes[s.Variable] = "List of " + assocQN
 			}
 		}
 	} else {

mdl/executor/cmd_microflows_builder_retrieve_reverse_test.go

@@ -12,7 +12,12 @@ import (
 	"github.com/mendixlabs/mxcli/sdk/microflows"
 )
 
-func TestAddRetrieveAction_ReverseReferenceOwnerBothUsesDatabaseSource(t *testing.T) {
+// A reverse Reference traversal (`$Child/Sample.Parent_Child`) is an in-memory
+// retrieve-by-association, NOT a database retrieve — Mendix returns a list and
+// mxbuild accepts it. Rewriting it to a DatabaseRetrieveSource made memory and
+// database retrieves indistinguishable on round-trip (issue #726). The output
+// variable is still typed as a list.
+func TestAddRetrieveAction_ReverseReferenceOwnerBothUsesAssociationSource(t *testing.T) {
 	fb := newRetrieveAssociationFlowBuilder(domainmodel.AssociationOwnerBoth)
 	fb.varTypes["Child"] = "Sample.Child"
 	fb.listInputVariables = map[string]bool{"Parents": true}
@@ -24,12 +29,12 @@ func TestAddRetrieveAction_ReverseReferenceOwnerBothUsesDatabaseSource(t *testin
 	})
 
 	action := onlyRetrieveAction(t, fb)
-	source, ok := action.Source.(*microflows.DatabaseRetrieveSource)
+	source, ok := action.Source.(*microflows.AssociationRetrieveSource)
 	if !ok {
-		t.Fatalf("owner-both reverse retrieve source = %T, want DatabaseRetrieveSource", action.Source)
+		t.Fatalf("owner-both reverse retrieve source = %T, want AssociationRetrieveSource", action.Source)
 	}
-	if source.EntityQualifiedName != "Sample.Parent" || source.XPathConstraint != "[Sample.Parent_Child = $Child]" {
-		t.Fatalf("database source = %#v", source)
+	if source.StartVariable != "Child" || source.AssociationQualifiedName != "Sample.Parent_Child" {
+		t.Fatalf("association source = %#v", source)
 	}
 	if got := fb.varTypes["Parents"]; got != "List of Sample.Parent" {
 		t.Fatalf("result var type = %q, want List of Sample.Parent", got)
@@ -60,7 +65,7 @@ func TestAddRetrieveAction_ReverseReferenceOwnerBothObjectUsagePreservesAssociat
 	}
 }
 
-func TestAddRetrieveAction_ReverseReferenceDefaultOwnerUsesDatabaseSource(t *testing.T) {
+func TestAddRetrieveAction_ReverseReferenceDefaultOwnerUsesAssociationSource(t *testing.T) {
 	fb := newRetrieveAssociationFlowBuilder(domainmodel.AssociationOwnerDefault)
 	fb.varTypes["Child"] = "Sample.Child"
 
@@ -71,12 +76,12 @@ func TestAddRetrieveAction_ReverseReferenceDefaultOwnerUsesDatabaseSource(t *tes
 	})
 
 	action := onlyRetrieveAction(t, fb)
-	source, ok := action.Source.(*microflows.DatabaseRetrieveSource)
+	source, ok := action.Source.(*microflows.AssociationRetrieveSource)
 	if !ok {
-		t.Fatalf("default-owner reverse retrieve source = %T, want DatabaseRetrieveSource", action.Source)
+		t.Fatalf("default-owner reverse retrieve source = %T, want AssociationRetrieveSource", action.Source)
 	}
-	if source.EntityQualifiedName != "Sample.Parent" || source.XPathConstraint != "[Sample.Parent_Child = $Child]" {
-		t.Fatalf("database source = %#v", source)
+	if source.StartVariable != "Child" || source.AssociationQualifiedName != "Sample.Parent_Child" {
+		t.Fatalf("association source = %#v", source)
 	}
 	if got := fb.varTypes["Parents"]; got != "List of Sample.Parent" {
 		t.Fatalf("result var type = %q, want List of Sample.Parent", got)
@@ -190,7 +195,7 @@ func TestBuildFlowGraph_ReverseReferenceOwnerBothAttributeUsagePreservesAssociat
 	}
 }
 
-func TestBuildFlowGraph_ReverseReferenceOwnerBothLoopUsageUsesDatabaseSource(t *testing.T) {
+func TestBuildFlowGraph_ReverseReferenceOwnerBothLoopUsageUsesAssociationSource(t *testing.T) {
 	fb := newRetrieveAssociationFlowBuilder(domainmodel.AssociationOwnerBoth)
 	fb.posX = 200
 	fb.posY = 200
@@ -212,12 +217,12 @@ func TestBuildFlowGraph_ReverseReferenceOwnerBothLoopUsageUsesDatabaseSource(t *
 	fb.buildFlowGraph(stmts, nil)
 
 	action := firstRetrieveAction(t, fb)
-	source, ok := action.Source.(*microflows.DatabaseRetrieveSource)
+	source, ok := action.Source.(*microflows.AssociationRetrieveSource)
 	if !ok {
-		t.Fatalf("owner-both loop usage source = %T, want DatabaseRetrieveSource", action.Source)
+		t.Fatalf("owner-both loop usage source = %T, want AssociationRetrieveSource", action.Source)
 	}
-	if source.EntityQualifiedName != "Sample.Parent" || source.XPathConstraint != "[Sample.Parent_Child = $Child]" {
-		t.Fatalf("database source = %#v", source)
+	if source.StartVariable != "Child" || source.AssociationQualifiedName != "Sample.Parent_Child" {
+		t.Fatalf("association source = %#v", source)
 	}
 	if got := fb.varTypes["Parents"]; got != "List of Sample.Parent" {
 		t.Fatalf("result var type = %q, want List of Sample.Parent", got)

mdl/executor/cmd_microflows_format_action.go

@@ -396,10 +396,13 @@ func formatAction(
 				entityName = "Entity"
 			}
 
-			if startVar, assocName, ok := parseReverseAssociationRetrieve(ctx, dbSource, entityName); ok {
-				return fmt.Sprintf("retrieve $%s from $%s/%s;", outputVar, startVar, assocName)
-			}
-
+			// A DatabaseRetrieveSource always renders in database form
+			// (`from Entity where …`). It must NOT be collapsed to the
+			// association shorthand `from $var/Assoc` even when its XPath is a
+			// reverse-association constraint — that shorthand is the in-memory
+			// retrieve-by-association form, and rendering a database retrieve
+			// that way makes the two indistinguishable and breaks round-trip
+			// (issue #726).
 			stmt := fmt.Sprintf("retrieve $%s from %s", outputVar, entityName)
 
 			if dbSource.XPathConstraint != "" {

mdl/executor/cmd_microflows_format_action_test.go

@@ -790,7 +790,11 @@ func TestFormatAction_Retrieve_Association(t *testing.T) {
 	}
 }
 
-func TestFormatAction_Retrieve_ReverseAssociationDatabaseSourceUsesCompactForm(t *testing.T) {
+// A DatabaseRetrieveSource always renders in database form, even when its XPath
+// is a reverse-association constraint — it must NOT collapse to the association
+// shorthand `from $var/Assoc`, which is the in-memory retrieve form. Rendering a
+// database retrieve that way made the two indistinguishable (issue #726).
+func TestFormatAction_Retrieve_ReverseAssociationDatabaseSourceUsesDatabaseForm(t *testing.T) {
 	e := newTestExecutor()
 	e.backend = reverseAssociationBackend(t)
 	action := &microflows.RetrieveAction{
@@ -803,7 +807,7 @@ func TestFormatAction_Retrieve_ReverseAssociationDatabaseSourceUsesCompactForm(t
 	}
 
 	got := e.formatAction(action, nil, nil)
-	want := "retrieve $Domains from $Runtime/SampleRuntime.Domain_Runtime;"
+	want := "retrieve $Domains from SampleRuntime.Domain\n    where SampleRuntime.Domain_Runtime = $Runtime;"
 	if got != want {
 		t.Errorf("got %q, want %q", got, want)
 	}