Remove vault.secrets.get from gateway; add workflow-based secret retrieval and cross-namespace E2E tests (#21660)

* Remove vault.secrets.get from gateway and add workflow-based secret retrieval tests

Remove the `vault.secrets.get` method from the gateway surface entirely:
- Remove `GetSupportedMethods()` dev-build conditional that exposed the method
- Remove `handleSecretsGet()` and `getEncryptionKeys()` from the gateway handler
- Remove the `MethodSecretsGet` case from the gateway-side handler and aggregator
- Remove the now-unused `capRegistry` field from `GatewayHandler`

Replace the commented-out gateway-based get test with a WASM workflow that calls
`runtime.GetSecret()`, and add cross-namespace E2E coverage proving that secrets
with the same ID in different namespaces are fully independent (create, get,
update, list, delete all scoped to their namespace).

Made-with: Cursor

* Harden negative test and registry update in vault E2E

- In the vaultsecret WASM workflow, check that the GetSecret error
  specifically contains "key does not exist" instead of accepting any
  error as proof the secret was deleted. This prevents config-propagation,
  transport, or decryption failures from masking real bugs.
- In updateVaultCapabilityConfigInRegistry, replace the fire-and-sleep
  pattern with sethClient.WaitMined + receipt status assertion so a
  reverted or stuck tx fails the test immediately instead of causing
  a downstream workflow flake.

Made-with: Cursor

* Fix CI failures: goimports, go.md, and vault DON lookup

- Fix goimports import ordering in v2_vault_don_test.go
- Add vaultsecret module to go.md dependency graph
- Fix updateVaultCapabilityConfigInRegistry to dynamically find the DON
  that has vault@1.0.0 instead of hardcoding "workflow-don", which fails
  in the workflow-gateway-capabilities topology where vault lives on
  "capabilities-don"

Made-with: Cursor

* fix: add MethodConfigs for vault capability to enable remote execution in multi-DON topology

In the workflow-gateway-capabilities topology, the vault capability runs
on a separate capabilities-don. Without MethodConfigs, the launcher
treats vault as a V1 capability and passes nil transmissionConfig. Since
secrets.go doesn't set Config on the CapabilityRequest, the V1 fallback
path fails with "cannot unwrap nil values.Map" when extracting
transmission config.

Adding RemoteExecutableConfig for the vault.secrets.get method ensures
the V2 Don2Don framework is used, which sets transmissionConfig from the
on-chain registry config rather than requiring it per-request.

* Address review feedback: parallelize vault tests, use seth.Decode, reduce sleeps
- Split ExecuteVaultTest into two parallel subtests (basic_crud +
  cross_namespace), each with its own per-test keys, ChIP sink, and
  channels. Follows the same pattern as HTTP Action tests. When
  parallelEnabled && fanoutEnabled, both subtests run concurrently.
- Replace manual deployerKey/deployerOpts + WaitMined + receipt check
  with a deployer seth client and sethClient.Decode() per Tofel's review.
- Remove redundant 30s "Vault DON ready" sleep.
- Reduce registry syncer wait from 30s to 15s (polls every 12s).
- Reduce allowlist sleep from 10s to 6s (polls every 5s).
- Remove unused consensus workflow deployment from vault test setup.

* Fix OwnershipLinkDoesNotExist error in vault subtests

Each subtest creates a new per-test key that hasn't completed the
linkOwner flow on the workflow registry. Call creworkflow.LinkOwner()
explicitly before any allowlistRequest operations.

Made-with: Cursor

Author: prashantkumar1982

core/capabilities/vault/gw_handler.go

@@ -2,15 +2,12 @@ package vault
 
 import (
 	"context"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"sort"
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
-	"google.golang.org/protobuf/proto"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
@@ -62,21 +59,19 @@ type GatewayHandler struct {
 	services.Service
 	eng *services.Engine
 
-	capRegistry      core.CapabilitiesRegistry
 	secretsService   vaulttypes.SecretsService
 	gatewayConnector gatewayConnector
 	lggr             logger.Logger
 	metrics          *metrics
 }
 
-func NewGatewayHandler(capabilitiesRegistry core.CapabilitiesRegistry, secretsService vaulttypes.SecretsService, connector gatewayConnector, lggr logger.Logger) (*GatewayHandler, error) {
+func NewGatewayHandler(secretsService vaulttypes.SecretsService, connector gatewayConnector, lggr logger.Logger) (*GatewayHandler, error) {
 	metrics, err := newMetrics()
 	if err != nil {
 		return nil, fmt.Errorf("failed to create metrics: %w", err)
 	}
 
 	gh := &GatewayHandler{
-		capRegistry:      capabilitiesRegistry,
 		secretsService:   secretsService,
 		gatewayConnector: connector,
 		lggr:             lggr.Named(HandlerName),
@@ -109,7 +104,7 @@ func (h *GatewayHandler) ID(ctx context.Context) (string, error) {
 }
 
 func (h *GatewayHandler) Methods() []string {
-	return vaulttypes.GetSupportedMethods(h.lggr)
+	return vaulttypes.Methods
 }
 
 func (h *GatewayHandler) HandleGatewayMessage(ctx context.Context, gatewayID string, req *jsonrpc.Request[json.RawMessage]) (err error) {
@@ -119,8 +114,6 @@ func (h *GatewayHandler) HandleGatewayMessage(ctx context.Context, gatewayID str
 	switch req.Method {
 	case vaulttypes.MethodSecretsCreate:
 		response = h.handleSecretsCreate(ctx, gatewayID, req)
-	case vaulttypes.MethodSecretsGet:
-		response = h.handleSecretsGet(ctx, gatewayID, req)
 	case vaulttypes.MethodSecretsUpdate:
 		response = h.handleSecretsUpdate(ctx, gatewayID, req)
 	case vaulttypes.MethodSecretsDelete:
@@ -183,51 +176,6 @@ func (h *GatewayHandler) handleSecretsUpdate(ctx context.Context, gatewayID stri
 	return jsonResponse
 }
 
-func (h *GatewayHandler) handleSecretsGet(ctx context.Context, gatewayID string, req *jsonrpc.Request[json.RawMessage]) *jsonrpc.Response[json.RawMessage] {
-	var request vaultcommon.GetSecretsRequest
-	if err := json.Unmarshal(*req.Params, &request); err != nil {
-		return h.errorResponse(ctx, gatewayID, req, api.UserMessageParseError, err)
-	}
-	encryptionKeys, err := h.getEncryptionKeys(ctx)
-	if err != nil {
-		return h.errorResponse(ctx, gatewayID, req, api.FatalError, err)
-	}
-	getSecretsRequest := vaultcommon.GetSecretsRequest{}
-	for _, reqItem := range request.Requests {
-		getSecretsRequest.Requests = append(getSecretsRequest.Requests, &vaultcommon.SecretRequest{
-			Id: &vaultcommon.SecretIdentifier{
-				Owner:     reqItem.Id.Owner,
-				Namespace: reqItem.Id.Namespace,
-				Key:       reqItem.Id.Key,
-			},
-			EncryptionKeys: encryptionKeys,
-		})
-	}
-	vaultCapResponse, err := h.secretsService.GetSecrets(ctx, req.ID, &getSecretsRequest)
-	if err != nil {
-		return h.errorResponse(ctx, gatewayID, req, api.FatalError, err)
-	}
-
-	vaultResponseProto := &vaultcommon.GetSecretsResponse{}
-	err = proto.Unmarshal(vaultCapResponse.Payload, vaultResponseProto)
-	if err != nil {
-		h.lggr.Errorf("Debugging: handleSecretsCreate failed to unmarshal response: %s. Payload was: %s", err.Error(), string(vaultCapResponse.Payload))
-		return h.errorResponse(ctx, gatewayID, req, api.NodeReponseEncodingError, err)
-	}
-
-	vaultAPIResponseBytes, err := json.Marshal(vaultResponseProto)
-	if err != nil {
-		return h.errorResponse(ctx, gatewayID, req, api.NodeReponseEncodingError, err)
-	}
-	vaultAPIResponseJSON := json.RawMessage(vaultAPIResponseBytes)
-	return &jsonrpc.Response[json.RawMessage]{
-		Version: jsonrpc.JsonRpcVersion,
-		ID:      req.ID,
-		Method:  req.Method,
-		Result:  &vaultAPIResponseJSON,
-	}
-}
-
 func (h *GatewayHandler) handleSecretsDelete(ctx context.Context, gatewayID string, req *jsonrpc.Request[json.RawMessage]) *jsonrpc.Response[json.RawMessage] {
 	r := &vaultcommon.DeleteSecretsRequest{}
 	if err := json.Unmarshal(*req.Params, r); err != nil {
@@ -324,26 +272,6 @@ func (h *GatewayHandler) errorResponse(
 	}
 }
 
-// getEncryptionKeys retrieves the encryption keys of all members in the Workflow DON.
-func (h *GatewayHandler) getEncryptionKeys(ctx context.Context) ([]string, error) {
-	myNode, err := h.capRegistry.LocalNode(ctx)
-	if err != nil {
-		return nil, errors.New("failed to get local node from registry" + err.Error())
-	}
-
-	encryptionKeys := make([]string, 0, len(myNode.WorkflowDON.Members))
-	for _, peerID := range myNode.WorkflowDON.Members {
-		peerNode, err := h.capRegistry.NodeByPeerID(ctx, peerID)
-		if err != nil {
-			return nil, errors.New("failed to get node info for peerID: " + peerID.String() + " - " + err.Error())
-		}
-		encryptionKeys = append(encryptionKeys, hex.EncodeToString(peerNode.EncryptionPublicKey[:]))
-	}
-	// Sort the encryption keys to ensure consistent ordering across all nodes.
-	sort.Strings(encryptionKeys)
-	return encryptionKeys, nil
-}
-
 func toJSONResponse(vaultCapResponse *vaulttypes.Response, method string) (*jsonrpc.Response[json.RawMessage], error) {
 	vaultResponseBytes, err := vaultCapResponse.ToJSONRPCResult()
 	if err != nil {

core/capabilities/vault/gw_handler_test.go

@@ -11,7 +11,6 @@ import (
 
 	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
 	jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
-	core_mocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks"
 	vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
 	vaulttypesmocks "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes/mocks"
@@ -167,11 +166,10 @@ func TestGatewayHandler_HandleGatewayMessage(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			secretsService := vaulttypesmocks.NewSecretsService(t)
 			gwConnector := connector_mocks.NewGatewayConnector(t)
-			capRegistry := core_mocks.NewCapabilitiesRegistry(t)
 
 			tt.setupMocks(secretsService, gwConnector)
 
-			handler, err := vaultcap.NewGatewayHandler(capRegistry, secretsService, gwConnector, lggr)
+			handler, err := vaultcap.NewGatewayHandler(secretsService, gwConnector, lggr)
 			require.NoError(t, err)
 
 			err = handler.HandleGatewayMessage(ctx, "gateway-1", tt.request)
@@ -191,19 +189,18 @@ func TestGatewayHandler_Lifecycle(t *testing.T) {
 
 	secretsService := vaulttypesmocks.NewSecretsService(t)
 	gwConnector := connector_mocks.NewGatewayConnector(t)
-	capRegistry := core_mocks.NewCapabilitiesRegistry(t)
 
-	handler, err := vaultcap.NewGatewayHandler(capRegistry, secretsService, gwConnector, lggr)
+	handler, err := vaultcap.NewGatewayHandler(secretsService, gwConnector, lggr)
 	require.NoError(t, err)
 
 	t.Run("start", func(t *testing.T) {
-		gwConnector.On("AddHandler", mock.Anything, vaulttypes.GetSupportedMethods(lggr), handler).Return(nil).Once()
+		gwConnector.On("AddHandler", mock.Anything, vaulttypes.Methods, handler).Return(nil).Once()
 		err := handler.Start(ctx)
 		require.NoError(t, err)
 	})
 
 	t.Run("close", func(t *testing.T) {
-		gwConnector.On("RemoveHandler", mock.Anything, vaulttypes.GetSupportedMethods(lggr)).Return(nil).Once()
+		gwConnector.On("RemoveHandler", mock.Anything, vaulttypes.Methods).Return(nil).Once()
 		err := handler.Close()
 		require.NoError(t, err)
 	})

core/capabilities/vault/vaulttypes/types.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"slices"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -15,8 +14,6 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/keystore/corekeys/ocr2key"
 	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
-	"github.com/smartcontractkit/chainlink-common/pkg/logger"
-	"github.com/smartcontractkit/chainlink/v2/core/build"
 )
 
 var DefaultNamespace = "main"
@@ -38,27 +35,12 @@ const (
 	MaxBatchSize = 10
 )
 
-var (
-	// MethodSecretsGet is intentionally omitted from this list, as it is not exposed
-	// to external clients, but rather used internally by the Workflow DON.
-	Methods = []string{
-		MethodSecretsCreate,
-		MethodSecretsUpdate,
-		MethodSecretsDelete,
-		MethodSecretsList,
-		MethodPublicKeyGet,
-	}
-)
-
-func GetSupportedMethods(lggr logger.Logger) []string {
-	methods := slices.Clone(Methods)
-	if build.IsDev() {
-		// Allow secrets get in non-prod environments for testing purposes
-		// This should never be enabled in production
-		methods = append(methods, MethodSecretsGet)
-		lggr.Warnw("enabling vault.secrets.get method since it is not a production build", "build-mode", build.Mode())
-	}
-	return methods
+var Methods = []string{
+	MethodSecretsCreate,
+	MethodSecretsUpdate,
+	MethodSecretsDelete,
+	MethodSecretsList,
+	MethodPublicKeyGet,
 }
 
 // SignedOCRResponse is the response format for OCR signed reports, as returned by the Vault DON.

core/services/gateway/handlers/vault/aggregator.go

@@ -139,11 +139,6 @@ func (a *baseAggregator) validateUsingSignatures(don capabilities.DON, nodes []c
 		return nil, errors.New("response result and error both are is nil: cannot validate signatures")
 	}
 
-	if resp.Method == vaulttypes.MethodSecretsGet {
-		// SecretsGet responses are not signed.
-		return resp, errors.New("cannot validate signatures for Get requests")
-	}
-
 	r := &vaulttypes.SignedOCRResponse{}
 	err := json.Unmarshal(*resp.Result, r)
 	if err != nil {

core/services/gateway/handlers/vault/aggregator_test.go

@@ -99,7 +99,7 @@ func newMessage(t *testing.T) *jsonrpc.Response[json.RawMessage] {
 	return &jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  (*json.RawMessage)(&rawResp),
 	}
 }
@@ -119,7 +119,7 @@ func TestAggregator_Valid_FallsBackToQuorum(t *testing.T) {
 	currResp := jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  (*json.RawMessage)(nil),
 		Error: &jsonrpc.WireError{
 			Code:    123,
@@ -180,7 +180,7 @@ func TestAggregator_InsufficientResponses(t *testing.T) {
 	currResp := jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  &rm,
 	}
 	responses := map[string]jsonrpc.Response[json.RawMessage]{
@@ -206,21 +206,21 @@ func TestAggregator_QuorumUnobtainable(t *testing.T) {
 	resp1 := &jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  &rm1,
 	}
 	rm2 := json.RawMessage([]byte(`{"foo": "bar"}`))
 	resp2 := &jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  &rm2,
 	}
 	rm3 := json.RawMessage([]byte(`{"baz": "qux"}`))
 	resp3 := &jsonrpc.Response[json.RawMessage]{
 		Version: jsonrpc.JsonRpcVersion,
 		ID:      "1",
-		Method:  vaulttypes.MethodSecretsGet,
+		Method:  vaulttypes.MethodSecretsCreate,
 		Result:  &rm3,
 	}
 	responses := map[string]jsonrpc.Response[json.RawMessage]{

core/services/gateway/handlers/vault/handler.go

@@ -326,7 +326,7 @@ func (h *handler) removeExpiredRequests(ctx context.Context) {
 }
 
 func (h *handler) Methods() []string {
-	return vaulttypes.GetSupportedMethods(h.lggr)
+	return vaulttypes.Methods
 }
 
 func (h *handler) HandleLegacyUserMessage(_ context.Context, _ *api.Message, _ gwhandlers.Callback) error {
@@ -361,9 +361,6 @@ func (h *handler) HandleJSONRPCUserMessage(ctx context.Context, req jsonrpc.Requ
 		h.lggr.Debugw("returning cached public key response")
 		return h.handlePublicKeyGetSynchronously(ctx, req, publicKeyResponseBytes, callback)
 
-	case vaulttypes.MethodSecretsGet:
-		h.lggr.Errorw("Get requests not allowed", "requestID", req.ID)
-		return errors.New("get request not allowed")
 	}
 
 	isAuthorized, owner, err := h.requestAuthorizer.AuthorizeRequest(ctx, req)

core/services/ocr2/delegate.go

@@ -726,7 +726,7 @@ func (d *Delegate) newServicesVaultPlugin(
 	}
 	srvs = append(srvs, vaultCapability)
 
-	handler, err := vaultcap.NewGatewayHandler(capabilitiesRegistry, vaultCapability, gwconnector, d.lggr)
+	handler, err := vaultcap.NewGatewayHandler(vaultCapability, gwconnector, d.lggr)
 	if err != nil {
 		return nil, fmt.Errorf("failed to instantiate vault plugin: failed to create vault handler: %w", err)
 	}

go.md

@@ -486,6 +486,7 @@ flowchart LR
 	chainlink/system-tests/tests --> chainlink/system-tests/tests/smoke/cre/evmread
 	chainlink/system-tests/tests --> chainlink/system-tests/tests/smoke/cre/httpaction
 	chainlink/system-tests/tests --> chainlink/system-tests/tests/smoke/cre/solana/solwrite
+	chainlink/system-tests/tests --> chainlink/system-tests/tests/smoke/cre/vaultsecret
 	click chainlink/system-tests/tests href "https://github.com/smartcontractkit/chainlink"
 	chainlink/system-tests/tests/regression/cre/consensus --> cre-sdk-go/capabilities/scheduler/cron
 	click chainlink/system-tests/tests/regression/cre/consensus href "https://github.com/smartcontractkit/chainlink"
@@ -517,6 +518,8 @@ flowchart LR
 	chainlink/system-tests/tests/smoke/cre/solana/solwrite --> cre-sdk-go/capabilities/blockchain/solana
 	chainlink/system-tests/tests/smoke/cre/solana/solwrite --> cre-sdk-go/capabilities/scheduler/cron
 	click chainlink/system-tests/tests/smoke/cre/solana/solwrite href "https://github.com/smartcontractkit/chainlink"
+	chainlink/system-tests/tests/smoke/cre/vaultsecret --> cre-sdk-go/capabilities/scheduler/cron
+	click chainlink/system-tests/tests/smoke/cre/vaultsecret href "https://github.com/smartcontractkit/chainlink"
 	chainlink/v2 --> chainlink-automation
 	chainlink/v2 --> chainlink-ccv
 	chainlink/v2 --> chainlink-evm/contracts/cre/gobindings
@@ -590,6 +593,7 @@ flowchart LR
 		 chainlink/system-tests/tests/smoke/cre/evmread
 		 chainlink/system-tests/tests/smoke/cre/httpaction
 		 chainlink/system-tests/tests/smoke/cre/solana/solwrite
+		 chainlink/system-tests/tests/smoke/cre/vaultsecret
 		 chainlink/v2
 	end
 	click chainlink-repo href "https://github.com/smartcontractkit/chainlink"

system-tests/lib/cre/features/vault/vault.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"strconv"
+	"time"
 
 	"dario.cat/mergo"
 	"github.com/Masterminds/semver/v3"
@@ -13,6 +14,7 @@ import (
 	"github.com/pelletier/go-toml/v2"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"google.golang.org/protobuf/types/known/durationpb"
 
 	chainselectors "github.com/smartcontractkit/chain-selectors"
 	"github.com/smartcontractkit/smdkg/dkgocr/dkgocrtypes"
@@ -105,7 +107,8 @@ func (o *Vault) PreEnvStartup(
 			CapabilityType: 1, // ACTION
 		},
 		Config: &capabilitiespb.CapabilityConfig{
-			LocalOnly: don.HasOnlyLocalCapabilities(),
+			LocalOnly:     don.HasOnlyLocalCapabilities(),
+			MethodConfigs: vaultMethodConfigs(),
 		},
 	}}
 
@@ -385,6 +388,20 @@ func reportingPluginConfigOverride(vaultDKGOCR3Addr *common.Address, creEnv *cre
 	return cfgb, nil
 }
 
+func vaultMethodConfigs() map[string]*capabilitiespb.CapabilityMethodConfig {
+	return map[string]*capabilitiespb.CapabilityMethodConfig{
+		vaultprotos.MethodGetSecrets: {
+			RemoteConfig: &capabilitiespb.CapabilityMethodConfig_RemoteExecutableConfig{
+				RemoteExecutableConfig: &capabilitiespb.RemoteExecutableConfig{
+					RequestTimeout:            durationpb.New(2 * time.Minute),
+					ServerMaxParallelRequests: 10,
+					RequestHasherType:         capabilitiespb.RequestHasherType_Simple,
+				},
+			},
+		},
+	}
+}
+
 func EncryptSecret(secret, masterPublicKeyStr string, owner common.Address) (string, error) {
 	masterPublicKey := tdh2easy.PublicKey{}
 	masterPublicKeyBytes, err := hex.DecodeString(masterPublicKeyStr)