(fix): release workflow binary memory after use during init (#21525)

Author: justinkaseman

core/services/workflows/syncer/v2/handler.go

@@ -757,6 +757,8 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp
 	if err != nil {
 		return fmt.Errorf("failed to decode workflow spec binary: %w", err)
 	}
+	// Free the hex-encoded binary string as it is not needed beyond this decode
+	spec.Workflow = ""
 
 	// Workflow Registry version >2 no longer handles secrets
 	secretsURL := ""
@@ -767,7 +769,8 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp
 	if err != nil {
 		return fmt.Errorf("failed to decode owner: %w", err)
 	}
-	hash, err := pkgworkflows.GenerateWorkflowID(ownerBytes, spec.WorkflowName, decodedBinary, []byte(spec.Config), secretsURL)
+	configBytes := []byte(spec.Config)
+	hash, err := pkgworkflows.GenerateWorkflowID(ownerBytes, spec.WorkflowName, decodedBinary, configBytes, secretsURL)
 	if err != nil {
 		return fmt.Errorf("failed to generate workflow id: %w", err)
 	}
@@ -790,16 +793,19 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp
 	// before emitting the workflowActivated event, ensuring the event accurately reflects deployment status.
 	initDone := make(chan error, 1)
 
-	engine, err := h.engineFactory(
-		ctx,
-		spec.WorkflowID,
-		spec.WorkflowOwner,
-		workflowName,
-		spec.WorkflowTag,
-		[]byte(spec.Config),
-		decodedBinary,
-		initDone,
-	)
+	// Scope the engineFactory call so that decodedBinary goes out of scope immediately after the factory returns
+	engine, err := func() (services.Service, error) {
+		return h.engineFactory(
+			ctx,
+			spec.WorkflowID,
+			spec.WorkflowOwner,
+			workflowName,
+			spec.WorkflowTag,
+			configBytes,
+			decodedBinary,
+			initDone,
+		)
+	}()
 	if err != nil {
 		return fmt.Errorf("failed to create workflow engine: %w", err)
 	}

core/services/workflows/syncer/v2/workflow_registry.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"maps"
 	"math/big"
+	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -445,14 +446,14 @@ func (w *workflowRegistry) generateReconciliationEvents(
 ) ([]*reconciliationEvent, error) {
 	var events []*reconciliationEvent
 	localHead := toLocalHead(head)
-	// workflowMetadataMap is only used for lookups; disregard when reading the state machine.
-	workflowMetadataMap := make(map[string]WorkflowMetadataView)
+	// workflowMetadataIDs is a set of workflow IDs present in this tick's metadata
+	workflowMetadataIDs := make(map[string]struct{}, len(workflowMetadata))
 	for _, wfMeta := range workflowMetadata {
-		workflowMetadataMap[wfMeta.WorkflowID.Hex()] = wfMeta
+		workflowMetadataIDs[wfMeta.WorkflowID.Hex()] = struct{}{}
 	}
 
-	// Keep track of which of the engines in the engineRegistry have been touched
-	workflowsSeen := map[string]bool{}
+	// Keep track of which of the engines in the engineRegistry have been touched.
+	workflowsSeen := make(map[string]bool, len(workflowMetadata))
 	for _, wfMeta := range workflowMetadata {
 		id := wfMeta.WorkflowID.Hex()
 		engineFound := w.engineRegistry.Contains(wfMeta.WorkflowID)
@@ -596,7 +597,7 @@ func (w *workflowRegistry) generateReconciliationEvents(
 	// the workflow no longer exists in this source's metadata
 	for id, event := range pendingEvents {
 		if event.Name == WorkflowActivated {
-			if _, ok := workflowMetadataMap[event.Data.(WorkflowActivatedEvent).WorkflowID.Hex()]; !ok {
+			if _, ok := workflowMetadataIDs[event.Data.(WorkflowActivatedEvent).WorkflowID.Hex()]; !ok {
 				delete(pendingEvents, id)
 			}
 		}
@@ -818,6 +819,12 @@ func (w *workflowRegistry) syncUsingReconciliationStrategy(ctx context.Context)
 				}
 				wg.Wait()
 
+				// prompt the GC to reclaim transient allocations from event handling
+				// that would otherwise be delayed because the dominant CGo/wasmtime memory is invisible to the Go GC
+				if dispatched > 0 {
+					runtime.GC()
+				}
+
 				batchDuration := time.Since(batchStart)
 				w.metrics.recordReconcileBatch(ctx, sourceName, dispatched, batchDuration)
 				if backoffCount > 0 {