[DX-3791] devenv tests Docker logs panics scan (#22021)

* use Docker logs fanout instead of reading logs in each function call

* add scaning for panics to automation test

* for all devenv tests scan logs for panics or errors + save them (in case of soak tests save only on failure)

* remove redundant wrapper

* add allowed error messages

* fix compile

* optimise clean up for subtests

* fix lints

* CR changes

* gomodtidy

* CR changes

Author: Tofel

core/scripts/go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260119171452-39c98c3b33cd
 	github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260409211238-5b99921cbc7c
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.17
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter v1.0.2
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.20
 	github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5

core/scripts/go.sum

@@ -27,7 +27,7 @@ require (
 	github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260318010722-59d4165024f1
 	github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20251211123524-f0c4fe7cfc0a
 	github.com/smartcontractkit/chainlink-protos/job-distributor v0.12.0
-	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16
+	github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.17
 	github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.14.9
 	github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5
 	github.com/smartcontractkit/chainlink-testing-framework/wasp v1.51.2

devenv/go.mod

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/api/types/container"
 	dfilter "github.com/docker/docker/api/types/filters"
@@ -71,6 +72,15 @@ func ScanLogs(l zerolog.Logger, settings ChainlinkNodeLogScannerSettings) error
 		return lErr
 	}
 
+	return ScanLogsFromStreams(l, settings, logStream)
+}
+
+func ScanLogsFromStreams(l zerolog.Logger, settings ChainlinkNodeLogScannerSettings, logStream map[string]io.ReadCloser) error {
+	if len(logStream) == 0 {
+		l.Info().Msg("No container logs found to scan")
+		return nil
+	}
+
 	verifyLogsGroup := &errgroup.Group{}
 	for _, stream := range logStream {
 		verifyLogsGroup.Go(func() error {
@@ -240,3 +250,30 @@ func DefaultSettings(extraAllowedMessages ...AllowedLogMessage) ChainlinkNodeLog
 		AllowedMessages: allowedMessages,
 	}
 }
+
+func CleanupContainerLogs(settings ChainlinkNodeLogScannerSettings) error {
+	logDir := fmt.Sprintf("%s-%d", framework.DefaultCTFLogsDir, time.Now().UnixNano())
+
+	return framework.StreamCTFContainerLogsFanout(
+		framework.LogStreamConsumer{
+			Name: "scan-logs",
+			Consume: func(logStreams map[string]io.ReadCloser) error {
+				return ScanLogsFromStreams(framework.L, settings, logStreams)
+			},
+		},
+		framework.LogStreamConsumer{
+			Name: "save-container-logs",
+			Consume: func(logStreams map[string]io.ReadCloser) error {
+				_, saveErr := framework.SaveContainerLogsFromStreams(logDir, logStreams)
+				return saveErr
+			},
+		},
+		framework.LogStreamConsumer{
+			Name: "print-panic-logs",
+			Consume: func(logStreams map[string]io.ReadCloser) error {
+				_ = framework.CheckContainersForPanicsFromStreams(logStreams, 100)
+				return nil
+			},
+		},
+	)
+}

devenv/go.sum

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 	"math"
 	"math/big"
 	"strconv"
@@ -61,8 +62,39 @@ func TestLoad(t *testing.T) {
 	}
 
 	t.Cleanup(func() {
-		_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, cErr)
+		scanErr := framework.StreamCTFContainerLogsFanout(
+			framework.LogStreamConsumer{
+				Name: "scan-logs",
+				Consume: func(logStreams map[string]io.ReadCloser) error {
+					return products.ScanLogsFromStreams(framework.L, products.DefaultSettings(), logStreams)
+				},
+			},
+			framework.LogStreamConsumer{
+				Name: "print-panic-logs",
+				Consume: func(logStreams map[string]io.ReadCloser) error {
+					_ = framework.CheckContainersForPanicsFromStreams(logStreams, 100)
+					return nil
+				},
+			},
+		)
+		t.Error("failed to scan Docker container logs:", scanErr)
+
+		if t.Failed() {
+			saveErr := framework.StreamCTFContainerLogsFanout(
+				framework.LogStreamConsumer{
+					Name: "save-container-logs",
+					Consume: func(logStreams map[string]io.ReadCloser) error {
+						_, saveErr := framework.SaveContainerLogsFromStreams(fmt.Sprintf("%s-%d", framework.DefaultCTFLogsDir, time.Now().UnixNano()), logStreams)
+						return saveErr
+					},
+				},
+			)
+			if saveErr != nil {
+				framework.L.Error().Err(saveErr).Msg("failed to save Docker container logs")
+			}
+		}
+		// check scanErr only after saving logs to ensure we don't miss any errors
+		require.NoError(t, scanErr, "failed to save Docker container logs")
 	})
 
 	for _, tc := range testCases {
@@ -78,15 +110,6 @@ func TestLoad(t *testing.T) {
 			fmt.Print(string(tcStr))
 			fmt.Println("--------------------------------")
 
-			// dangerous: takes a lot of time, if test runs for a long time
-			// t.Cleanup(func() {
-			// 	err := products.ScanLogs(l, products.DefaultSettings())
-			// 	require.NoError(t, err, "Found concerning logs in Chainlink Node logs")
-
-			// 	_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-			// 	require.NoError(t, cErr)
-			// })
-
 			outputFile := "../../env-out.toml"
 			in, err := de.LoadOutput[de.Cfg](outputFile)
 			require.NoError(t, err)

devenv/products/logs.go

@@ -2,7 +2,6 @@ package automation
 
 import (
 	"encoding/json"
-	"fmt"
 	"math/big"
 	"os"
 	"strconv"
@@ -43,8 +42,7 @@ func TestRegistry_2_0(t *testing.T) {
 }
 
 func TestRegistry_2_1(t *testing.T) {
-	// testNames := []string{"registry_2_1_conditional", "registry_2_1_logtrigger", "registry_2_1_with_mercury_v02", "registry_2_1_with_mercury_v03"}
-	testNames := []string{"registry_2_1_logtrigger"}
+	testNames := []string{"registry_2_1_conditional", "registry_2_1_logtrigger", "registry_2_1_with_mercury_v02", "registry_2_1_with_mercury_v03"}
 	for _, tc := range testNames {
 		basicAutomationTest(t, Testcase{
 			RegistryVersion:          contracts.RegistryVersion_2_1,
@@ -96,11 +94,8 @@ func basicAutomationTest(t *testing.T, testcase Testcase) {
 	l.Info().Msg("Running test " + testcase.Name + " with registry version " + testcase.RegistryVersion.String())
 
 	t.Cleanup(func() {
-		err := products.ScanLogs(l, products.DefaultSettings())
-		require.NoError(t, err, "Found concerning logs in Chainlink Node logs")
-
-		_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, cErr)
+		cleanupErr := products.CleanupContainerLogs(products.DefaultSettings())
+		require.NoError(t, cleanupErr, "failed to process cleanup container logs")
 	})
 
 	outputFile := "../../env-out.toml"

devenv/tests/automation/load_test.go

@@ -1,14 +1,12 @@
 package cron
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/clclient"
 	de "github.com/smartcontractkit/chainlink/devenv"
 	"github.com/smartcontractkit/chainlink/devenv/products"
@@ -22,8 +20,8 @@ func TestSmoke(t *testing.T) {
 	pdConfig, err := products.LoadOutput[cron.Configurator](outputFile)
 	require.NoError(t, err)
 	t.Cleanup(func() {
-		_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, cErr)
+		cleanupErr := products.CleanupContainerLogs(products.DefaultSettings())
+		require.NoError(t, cleanupErr, "failed to process cleanup container logs")
 	})
 
 	cls, err := clclient.New(in.NodeSets[0].Out.CLNodes)

devenv/tests/automation/smoke_test.go

@@ -1,7 +1,6 @@
 package directrequest
 
 import (
-	"fmt"
 	"math/big"
 	"testing"
 	"time"
@@ -10,7 +9,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 
 	"github.com/smartcontractkit/chainlink-evm/gethwrappers/generated/test_api_consumer_wrapper"
-	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	de "github.com/smartcontractkit/chainlink/devenv"
 	"github.com/smartcontractkit/chainlink/devenv/products"
 	"github.com/smartcontractkit/chainlink/devenv/products/directrequest"
@@ -27,8 +25,8 @@ func TestSmoke(t *testing.T) {
 	productCfg, err := products.LoadOutput[directrequest.Configurator](outputFile)
 	require.NoError(t, err)
 	t.Cleanup(func() {
-		_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, cErr)
+		cleanupErr := products.CleanupContainerLogs(products.DefaultSettings())
+		require.NoError(t, cleanupErr, "failed to process cleanup container logs")
 	})
 
 	c, auth, _, err := products.ETHClient(

devenv/tests/cron/smoke_test.go

@@ -1,14 +1,13 @@
 package features
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/clclient/gql/client"
 	de "github.com/smartcontractkit/chainlink/devenv"
+	"github.com/smartcontractkit/chainlink/devenv/products"
 )
 
 func TestMultipleJobDistributors(t *testing.T) {
@@ -18,8 +17,8 @@ func TestMultipleJobDistributors(t *testing.T) {
 	node := in.NodeSets[0].Out.CLNodes[0].Node
 
 	t.Cleanup(func() {
-		_, cErr := framework.SaveContainerLogs(fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()))
-		require.NoError(t, cErr)
+		cleanupErr := products.CleanupContainerLogs(products.DefaultSettings())
+		require.NoError(t, cleanupErr, "failed to process cleanup container logs")
 	})
 
 	c, err := client.NewWithContext(t.Context(), node.ExternalURL, client.Credentials{