[codex] Add Vault JWT local CRE test coverage

Author: prashantkumar1982

.github/workflows/cre-system-tests.yaml

@@ -160,7 +160,9 @@ jobs:
       # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments
       name: integration
       deployment: false
-    timeout-minutes: 10
+    # Bucket B now runs both legacy Vault auth and JWT auth against the same topology.
+    # Give that matrix entry a larger budget without slowing the rest of the suite.
+    timeout-minutes: ${{ matrix.tests.test_name == 'Test_CRE_V2_Suite_Bucket_B' && 16 || 10 }}
     env:
       ENABLE_AUTO_QUARANTINE: "true"
       # override Chip Ingress and Chip Config images with remote images. We have added this env var here, instead of the "Start local CRE" step, because
@@ -328,9 +330,12 @@ jobs:
         continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }}
         env:
           TEST_NAME: ${{ matrix.tests.test_name }}
-          TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes)
+          TEST_TIMEOUT: ${{ matrix.tests.test_name == 'Test_CRE_V2_Suite_Bucket_B' && '12m' || '7m' }}
           RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI
           TOPOLOGY_NAME: ${{ matrix.tests.topology }}
+          CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.22.1"
+          CTF_CHAINLINK_IMAGE: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
+          CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
           GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }} # to avoid rate limiting when downloading protobuf files from GitHub
           PARALLEL_COUNT: "10"
           CRE_TEST_PARALLEL_ENABLED: "true"

core/capabilities/vault/authorizer_test.go

@@ -11,29 +11,20 @@ import (
 
 	vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
 	jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
-	"github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings"
-	"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
 	vault "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault"
 	vaultmocks "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/mocks"
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 )
 
-func testLimitsFactory() limits.Factory {
-	return limits.Factory{Settings: cresettings.DefaultGetter}
-}
-
-func TestAuthorizer_RejectsJWTBasedAuthWhenDisabled(t *testing.T) {
+func TestAuthorizer_RejectsJWTBasedAuthWhenUnavailable(t *testing.T) {
 	params, err := json.Marshal(vaultcommon.CreateSecretsRequest{})
 	require.NoError(t, err)
 
 	allowListBasedAuth := vaultmocks.NewAuthorizer(t)
 	allowListBasedAuth.EXPECT().AuthorizeRequest(mock.Anything, mock.Anything).Maybe()
 
-	jwtBasedAuth, err := vault.NewJWTBasedAuth(vault.JWTBasedAuthConfig{}, testLimitsFactory(), logger.TestLogger(t), vault.WithDisabledJWTBasedAuth())
-	require.NoError(t, err)
-
-	a := vault.NewAuthorizer(allowListBasedAuth, jwtBasedAuth, logger.TestLogger(t))
+	a := vault.NewAuthorizer(allowListBasedAuth, nil, logger.TestLogger(t))
 
 	authResult, err := a.AuthorizeRequest(t.Context(), jsonrpc.Request[json.RawMessage]{
 		ID:     "1",
@@ -42,7 +33,7 @@ func TestAuthorizer_RejectsJWTBasedAuthWhenDisabled(t *testing.T) {
 		Auth:   "jwt-token",
 	})
 	require.Nil(t, authResult)
-	require.ErrorContains(t, err, "JWTBasedAuth is disabled")
+	require.ErrorContains(t, err, "JWTBasedAuth is nil")
 	allowListBasedAuth.AssertNotCalled(t, "AuthorizeRequest", mock.Anything, mock.Anything)
 }
 
@@ -126,10 +117,7 @@ func TestAuthorizer_RejectsAllowListBasedAuthReplay(t *testing.T) {
 	req := jsonrpc.Request[json.RawMessage]{ID: "1", Method: vaulttypes.MethodSecretsCreate}
 	allowListBasedAuth.EXPECT().AuthorizeRequest(mock.Anything, req).Return(vault.NewAuthResult("", "0xabc", "digest-1", time.Now().Add(time.Minute).Unix()), nil).Twice()
 
-	jwtBasedAuth, err := vault.NewJWTBasedAuth(vault.JWTBasedAuthConfig{}, testLimitsFactory(), logger.TestLogger(t), vault.WithDisabledJWTBasedAuth())
-	require.NoError(t, err)
-
-	a := vault.NewAuthorizer(allowListBasedAuth, jwtBasedAuth, logger.TestLogger(t))
+	a := vault.NewAuthorizer(allowListBasedAuth, nil, logger.TestLogger(t))
 
 	authResult, err := a.AuthorizeRequest(t.Context(), req)
 	require.NoError(t, err)

core/capabilities/vault/gw_handler.go

@@ -60,6 +60,7 @@ type gatewayConnector interface {
 
 type gatewayHandlerConfig struct {
 	authorizer Authorizer
+	auth0      *Auth0Config
 }
 
 // GatewayHandlerOption customizes GatewayHandler construction for tests and future auth extensions.
@@ -72,6 +73,13 @@ func WithAuthorizer(authorizer Authorizer) GatewayHandlerOption {
 	}
 }
 
+// WithJWTAuth0Config enables JWT-based request validation for the node-side Vault handler.
+func WithJWTAuth0Config(auth0 *Auth0Config) GatewayHandlerOption {
+	return func(cfg *gatewayHandlerConfig) {
+		cfg.auth0 = auth0
+	}
+}
+
 // GatewayHandler serves Vault requests received from the gateway on the node side.
 type GatewayHandler struct {
 	services.Service
@@ -93,12 +101,21 @@ func NewGatewayHandler(secretsService vaulttypes.SecretsService, connector gatew
 	}
 	if cfg.authorizer == nil {
 		allowListBasedAuth := NewAllowListBasedAuth(lggr, workflowRegistrySyncer)
-		jwtBasedAuth, err := NewJWTBasedAuth(JWTBasedAuthConfig{}, limitsFactory, lggr, WithDisabledJWTBasedAuth())
-		if err != nil {
-			return nil, fmt.Errorf("failed to create JWTBasedAuth: %w", err)
+		var jwtBasedAuth Authorizer
+		var jwtAuthService services.Service
+		if cfg.auth0 != nil {
+			var err error
+			jwtAuthService, err = NewJWTBasedAuth(JWTBasedAuthConfig{
+				IssuerURL: cfg.auth0.IssuerURL,
+				Audience:  cfg.auth0.Audience,
+			}, limitsFactory, lggr)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create JWTBasedAuth: %w", err)
+			}
+			jwtBasedAuth = jwtAuthService.(Authorizer)
 		}
 		cfg.authorizer = NewAuthorizer(allowListBasedAuth, jwtBasedAuth, lggr)
-		return newGatewayHandlerWithAuthorizer(secretsService, connector, cfg.authorizer, jwtBasedAuth, lggr)
+		return newGatewayHandlerWithAuthorizer(secretsService, connector, cfg.authorizer, jwtAuthService, lggr)
 	}
 	return newGatewayHandlerWithAuthorizer(secretsService, connector, cfg.authorizer, nil, lggr)
 }
@@ -227,6 +244,11 @@ func (h *GatewayHandler) authorizeAndPrefixRequest(ctx context.Context, req *jso
 		h.lggr.Errorw("failed to normalize gateway request for authorization", "method", req.Method, "requestID", originalRequestID, "error", err)
 		return nil, err
 	}
+	authReq, err := StripRequestIdentity(authReq)
+	if err != nil {
+		h.lggr.Errorw("failed to strip authorized identity fields before authorization", "method", req.Method, "requestID", originalRequestID, "error", err)
+		return nil, err
+	}
 
 	h.lggr.Debugw("authorizing gateway request", "method", req.Method, "requestID", originalRequestID)
 	authResult, err := h.authorizer.AuthorizeRequest(ctx, authReq)

core/capabilities/vault/gw_handler_test.go

@@ -121,6 +121,56 @@ func TestGatewayHandler_HandleGatewayMessage(t *testing.T) {
 			},
 			expectedError: false,
 		},
+		{
+			name: "success - create secrets strips forwarded identity before reauthorization",
+			setupMocks: func(ss *vaulttypesmocks.SecretsService, gc *connector_mocks.GatewayConnector, ra *vaultcapmocks.Authorizer) {
+				ra.EXPECT().AuthorizeRequest(mock.Anything, mock.MatchedBy(func(req jsonrpc.Request[json.RawMessage]) bool {
+					if req.Method != vaulttypes.MethodSecretsCreate || req.ID != "1" || req.Params == nil {
+						return false
+					}
+					parsed := &vaultcommon.CreateSecretsRequest{}
+					if err := json.Unmarshal(*req.Params, parsed); err != nil {
+						return false
+					}
+					return parsed.OrgId == "" && parsed.WorkflowOwner == ""
+				})).Return(authResult("org-1", "0xworkflow"), nil)
+				ss.EXPECT().CreateSecrets(mock.Anything, mock.MatchedBy(func(req *vaultcommon.CreateSecretsRequest) bool {
+					return len(req.EncryptedSecrets) == 1 &&
+						req.EncryptedSecrets[0].Id.Key == "test-secret" &&
+						req.EncryptedSecrets[0].Id.Owner == "org-1" &&
+						req.RequestId == "org-1"+vaulttypes.RequestIDSeparator+"1" &&
+						req.OrgId == "org-1" &&
+						req.WorkflowOwner == "0xworkflow"
+				})).Return(&vaulttypes.Response{ID: "test-secret"}, nil)
+
+				gc.On("SendToGateway", mock.Anything, "gateway-1", mock.MatchedBy(func(resp *jsonrpc.Response[json.RawMessage]) bool {
+					return resp.Error == nil
+				})).Return(nil)
+			},
+			request: &jsonrpc.Request[json.RawMessage]{
+				Method: vaulttypes.MethodSecretsCreate,
+				ID:     "org-1" + vaulttypes.RequestIDSeparator + "1",
+				Params: func() *json.RawMessage {
+					params, _ := json.Marshal(vaultcommon.CreateSecretsRequest{
+						RequestId:     "org-1" + vaulttypes.RequestIDSeparator + "1",
+						OrgId:         "org-1",
+						WorkflowOwner: "0xworkflow",
+						EncryptedSecrets: []*vaultcommon.EncryptedSecret{
+							{
+								Id: &vaultcommon.SecretIdentifier{
+									Key:   "test-secret",
+									Owner: "org-1",
+								},
+								EncryptedValue: "encrypted-value",
+							},
+						},
+					})
+					raw := json.RawMessage(params)
+					return &raw
+				}(),
+			},
+			expectedError: false,
+		},
 		{
 			name: "failure - service error",
 			setupMocks: func(ss *vaulttypesmocks.SecretsService, gc *connector_mocks.GatewayConnector, ra *vaultcapmocks.Authorizer) {
@@ -456,9 +506,6 @@ func TestGatewayHandler_HandleGatewayMessage(t *testing.T) {
 			secretsService := vaulttypesmocks.NewSecretsService(t)
 			gwConnector := connector_mocks.NewGatewayConnector(t)
 			allowListBasedAuth := vaultcapmocks.NewAuthorizer(t)
-			limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter}
-			jwtBasedAuth, err := vaultcap.NewJWTBasedAuth(vaultcap.JWTBasedAuthConfig{}, limitsFactory, lggr, vaultcap.WithDisabledJWTBasedAuth())
-			require.NoError(t, err)
 
 			tt.setupMocks(secretsService, gwConnector, allowListBasedAuth)
 
@@ -467,8 +514,8 @@ func TestGatewayHandler_HandleGatewayMessage(t *testing.T) {
 				gwConnector,
 				nil,
 				lggr,
-				limitsFactory,
-				vaultcap.WithAuthorizer(vaultcap.NewAuthorizer(allowListBasedAuth, jwtBasedAuth, lggr)),
+				limits.Factory{Settings: cresettings.DefaultGetter},
+				vaultcap.WithAuthorizer(vaultcap.NewAuthorizer(allowListBasedAuth, nil, lggr)),
 			)
 			require.NoError(t, err)
 
@@ -490,17 +537,14 @@ func TestGatewayHandler_Lifecycle(t *testing.T) {
 	secretsService := vaulttypesmocks.NewSecretsService(t)
 	gwConnector := connector_mocks.NewGatewayConnector(t)
 	allowListBasedAuth := vaultcapmocks.NewAuthorizer(t)
-	limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter}
-	jwtBasedAuth, err := vaultcap.NewJWTBasedAuth(vaultcap.JWTBasedAuthConfig{}, limitsFactory, lggr, vaultcap.WithDisabledJWTBasedAuth())
-	require.NoError(t, err)
 
 	handler, err := vaultcap.NewGatewayHandler(
 		secretsService,
 		gwConnector,
 		nil,
 		lggr,
-		limitsFactory,
-		vaultcap.WithAuthorizer(vaultcap.NewAuthorizer(allowListBasedAuth, jwtBasedAuth, lggr)),
+		limits.Factory{Settings: cresettings.DefaultGetter},
+		vaultcap.WithAuthorizer(vaultcap.NewAuthorizer(allowListBasedAuth, nil, lggr)),
 	)
 	require.NoError(t, err)
 
@@ -522,3 +566,26 @@ func TestGatewayHandler_Lifecycle(t *testing.T) {
 		assert.Equal(t, vaultcap.HandlerName, id)
 	})
 }
+
+func TestGatewayHandler_Lifecycle_DefaultAuthorizer_NoJWTConfig(t *testing.T) {
+	lggr := logger.TestLogger(t)
+	ctx := t.Context()
+
+	secretsService := vaulttypesmocks.NewSecretsService(t)
+	gwConnector := connector_mocks.NewGatewayConnector(t)
+
+	handler, err := vaultcap.NewGatewayHandler(
+		secretsService,
+		gwConnector,
+		nil,
+		lggr,
+		limits.Factory{Settings: cresettings.DefaultGetter},
+	)
+	require.NoError(t, err)
+
+	gwConnector.On("AddHandler", mock.Anything, vaulttypes.Methods, handler).Return(nil).Once()
+	require.NoError(t, handler.Start(ctx))
+
+	gwConnector.On("RemoveHandler", mock.Anything, vaulttypes.Methods).Return(nil).Once()
+	require.NoError(t, handler.Close())
+}

core/capabilities/vault/jwt_based_auth.go

@@ -37,6 +37,12 @@ const (
 	defaultHTTPTimeout         = 5 * time.Second
 )
 
+// Auth0Config captures the Vault JWT issuer settings shared by gateway and node handlers.
+type Auth0Config struct {
+	IssuerURL string `json:"issuerURL" toml:"issuerURL" yaml:"issuerURL"`
+	Audience  string `json:"audience" toml:"audience" yaml:"audience"`
+}
+
 // JWTBasedAuthConfig holds the configuration for JWTBasedAuth validation.
 type JWTBasedAuthConfig struct {
 	IssuerURL           string
@@ -84,7 +90,6 @@ type jwtBasedAuth struct {
 	jwksURL         string
 	refreshInterval time.Duration
 	authEnabledGate limits.GateLimiter
-	refreshEnabled  bool
 
 	mu            sync.RWMutex
 	keySet        *jsonWebKeySet
@@ -97,8 +102,7 @@ type jwtBasedAuth struct {
 }
 
 type jwtBasedAuthOptions struct {
-	authEnabledGate  limits.GateLimiter
-	skipConfigChecks bool
+	authEnabledGate limits.GateLimiter
 }
 
 // JWTBasedAuthOption customizes JWTBasedAuth construction without multiplying constructors.
@@ -111,14 +115,6 @@ func WithJWTBasedAuthGateLimiter(gateLimiter limits.GateLimiter) JWTBasedAuthOpt
 	}
 }
 
-// WithDisabledJWTBasedAuth makes the constructed JWTBasedAuth fail closed without requiring issuer config.
-func WithDisabledJWTBasedAuth() JWTBasedAuthOption {
-	return func(opts *jwtBasedAuthOptions) {
-		opts.authEnabledGate = limits.NewGateLimiter(false)
-		opts.skipConfigChecks = true
-	}
-}
-
 // NewJWTBasedAuth creates a JWTBasedAuth authorizer that verifies Auth0-issued JWTs
 // against the provider's JWKS endpoint. The JWKS is fetched lazily on first
 // use and refreshed on key-ID cache misses (rate-limited).
@@ -130,10 +126,10 @@ func NewJWTBasedAuth(cfg JWTBasedAuthConfig, limitsFactory limits.Factory, lggr
 	if options.authEnabledGate == nil {
 		options.authEnabledGate = newVaultJWTAuthEnabledGateLimiter(limitsFactory, lggr)
 	}
-	if !options.skipConfigChecks && cfg.IssuerURL == "" {
+	if cfg.IssuerURL == "" {
 		return nil, errors.New("issuer URL is required")
 	}
-	if !options.skipConfigChecks && cfg.Audience == "" {
+	if cfg.Audience == "" {
 		return nil, errors.New("audience is required")
 	}
 
@@ -156,7 +152,6 @@ func NewJWTBasedAuth(cfg JWTBasedAuthConfig, limitsFactory limits.Factory, lggr
 		jwksURL:         jwksURL,
 		refreshInterval: refreshInterval,
 		authEnabledGate: options.authEnabledGate,
-		refreshEnabled:  !options.skipConfigChecks,
 		httpClient:      httpClient,
 		lggr:            logger.Named(lggr, "VaultJWTBasedAuth"),
 	}
@@ -180,11 +175,6 @@ func newVaultJWTAuthEnabledGateLimiter(limitsFactory limits.Factory, lggr logger
 }
 
 func (v *jwtBasedAuth) start(context.Context) error {
-	if !v.refreshEnabled {
-		v.lggr.Debug("JWTBasedAuth periodic JWKS refresh disabled")
-		return nil
-	}
-
 	v.eng.GoTick(services.NewTicker(v.refreshInterval), func(ctx context.Context) {
 		if err := v.refreshJWKS(ctx); err != nil {
 			v.lggr.Warnw("periodic JWKS refresh failed", "error", err)
@@ -209,21 +199,21 @@ func (v *jwtBasedAuth) AuthorizeRequest(ctx context.Context, req jsonrpc.Request
 		return nil, errors.New("JWTBasedAuth is disabled")
 	}
 
-	requestDigest, err := req.Digest()
-	if err != nil {
-		v.lggr.Debugw("JWTBasedAuth failed to compute request digest", "method", req.Method, "requestID", req.ID, "error", err)
-		return nil, fmt.Errorf("failed to compute request digest: %w", err)
-	}
-
 	claims, err := v.validateToken(ctx, req.Auth)
 	if err != nil {
 		v.lggr.Debugw("JWTBasedAuth token validation failed", "method", req.Method, "requestID", req.ID, "error", err)
 		return nil, fmt.Errorf("invalid JWT auth token: %w", err)
 	}
 
+	requestDigest, err := req.Digest()
+	if err != nil {
+		v.lggr.Debugw("JWTBasedAuth failed to compute request digest", "method", req.Method, "requestID", req.ID, "orgID", claims.OrgID, "workflowOwner", claims.WorkflowOwner, "error", err)
+		return nil, fmt.Errorf("failed to compute request digest: %w", err)
+	}
+
 	if !strings.EqualFold(requestDigest, claims.RequestDigest) {
 		v.lggr.Debugw("JWTBasedAuth request digest mismatch", "method", req.Method, "requestID", req.ID, "orgID", claims.OrgID, "workflowOwner", claims.WorkflowOwner, "computedDigest", requestDigest, "claimedDigest", claims.RequestDigest)
-		return nil, errors.New("request digest mismatch")
+		return nil, fmt.Errorf("request digest mismatch: computed=%s claimed=%s", requestDigest, claims.RequestDigest)
 	}
 
 	v.lggr.Debugw("JWTBasedAuth authorization succeeded", "method", req.Method, "requestID", req.ID, "orgID", claims.OrgID, "workflowOwner", claims.WorkflowOwner, "digest", requestDigest, "expiresAt", claims.ExpiresAt.UTC().Unix())