HYPERFLEET-1147 - feat: add caller identity support for audit attribution

kuudori · kuudori · commit 5d8c6b8fb281 · 2026-06-08T18:59:36.000-05:00
Inject caller identity headers into all E2E API requests so tests work
against an API with identity enforcement enabled (production default).

- Add IdentityConfig (header/value/token) to config with CLI flags,
  env vars (HYPERFLEET_IDENTITY_*), and config.yaml support
- Inject identity via OpenAPI RequestEditorFn in helper.newHelper()
- Add HaveAuditIdentity matcher and ExpectedIdentity() helper
- Add created_by assertion in cluster creation, deleted_by in deletion
- Deploy API with identity_header enabled in local kind setup
- Update AGENTS.md, getting-started, and local-kind-setup docs
diff --git a/AGENTS.md b/AGENTS.md
@@ -43,6 +43,7 @@ Pre-flight order: `make check` then `make build`.
 | Labels | `pkg/labels/labels.go` |
 | Condition type constants | `pkg/client/constants.go` |
 | Config file | `configs/config.yaml` |
+| Identity config & transport | `pkg/config/config.go` (`IdentityConfig`), `pkg/helper/suite.go` (RequestEditorFn wiring) |
 
 ## Test Conventions
 
@@ -87,7 +88,7 @@ Eventually(h.PollNamespacesByPrefix(ctx, clusterID), timeout, h.Cfg.Polling.Inte
 
 Available pollers: `PollCluster`, `PollNodePool`, `PollClusterAdapterStatuses`, `PollNodePoolAdapterStatuses`, `PollClusterHTTPStatus`, `PollNodePoolHTTPStatus`, `PollNamespacesByPrefix`.
 
-Available matchers: `HaveResourceCondition`, `HaveAllAdaptersWithCondition`, `HaveAllAdaptersAtGeneration`.
+Available matchers: `HaveResourceCondition`, `HaveAllAdaptersWithCondition`, `HaveAllAdaptersAtGeneration`, `HaveAuditIdentity`.
 
 For one-off complex assertions, use `Eventually(func(g Gomega) { ... }).Should(Succeed())` with `g.Expect()` (not bare `Expect()`).
 
@@ -118,6 +119,39 @@ Use `ginkgo.By()` for major steps. **IMPORTANT:** Never use `ginkgo.By()` inside
 
 Always use config values: `h.Cfg.Timeouts.Cluster.Reconciled`, `h.Cfg.Timeouts.NodePool.Reconciled`, `h.Cfg.Timeouts.Adapter.Processing`, `h.Cfg.Polling.Interval`. Never hardcode durations.
 
+### Caller identity (audit attribution)
+
+The API enforces caller identity on mutating requests (production default). E2E tests must send identity headers — without them, mutating requests are rejected with `401 Unauthorized`.
+
+```bash
+# Header mode (simplest)
+HYPERFLEET_IDENTITY_HEADER=X-HyperFleet-Identity \
+HYPERFLEET_IDENTITY_VALUE=e2e-test@hyperfleet.local \
+./bin/hyperfleet-e2e test
+
+# Or via CLI flags
+./bin/hyperfleet-e2e test \
+  --identity-header X-HyperFleet-Identity \
+  --identity-value e2e-test@hyperfleet.local
+
+# JWT mode (for Prow CI with GCP SA)
+HYPERFLEET_IDENTITY_TOKEN=$TOKEN ./bin/hyperfleet-e2e test
+```
+
+Identity injection uses `openapi.WithRequestEditorFn` — headers are added to every request from the generated OpenAPI client. The `identity.value` must be email-formatted (the API validates `created_by` as email type).
+
+To verify audit fields in tests, use `h.ExpectedIdentity()` and `HaveAuditIdentity`:
+
+```go
+if expected := h.ExpectedIdentity(); expected != "" {
+    Expect(cluster).To(helper.HaveAuditIdentity(expected))
+}
+```
+
+When no identity is configured, `ExpectedIdentity()` returns `""` and audit assertions are skipped. However, if the API enforces identity, tests will fail with `401` before reaching any assertions.
+
+For local kind setup, `deploy-scripts/lib/api.sh` automatically deploys the API with `config.server.identity_header=X-HyperFleet-Identity` via helm.
+
 ## Boundaries
 
 ### DON'T
@@ -136,3 +170,5 @@ Always use config values: `h.Cfg.Timeouts.Cluster.Reconciled`, `h.Cfg.Timeouts.N
 - Config file path priority: `--config` flag > `HYPERFLEET_CONFIG` env > `./configs/config.yaml` auto-detect
 - Adapter names come from `h.Cfg.Adapters.Cluster` and `h.Cfg.Adapters.NodePool` at runtime — never hardcode adapter names. Values in `configs/config.yaml` (e.g., `cl-namespace`) override compiled defaults in `pkg/config/defaults.go` (e.g., `clusters-namespace`)
 - `e2e-ci` Makefile target sets `TESTDATA_DIR` to absolute path and writes JUnit XML to `output/`
+- Identity config is optional — when all identity fields are empty, no headers are injected and audit assertions are skipped. The `identity.value` must be email-formatted (e.g., `user@domain.local`) because the API's `created_by` field is typed as `openapi_types.Email`
+- Identity header name must match the API's `server.identity_header` config — conventional value is `X-HyperFleet-Identity`
diff --git a/cmd/hyperfleet-e2e/main.go b/cmd/hyperfleet-e2e/main.go
@@ -30,18 +30,22 @@ func init() {
 	pfs.StringVar(&logLevel, "log-level", config.DefaultLogLevel, "Log level (debug, info, warn, error)")
 	pfs.StringVar(&logFormat, "log-format", config.DefaultLogFormat, "Log format (text, json)")
 	pfs.StringVar(&logOutput, "log-output", config.DefaultLogOutput, "Log output (stdout, stderr)")
+	pfs.StringVar(&identityHeader, "identity-header", "", "HTTP header name for caller identity (e.g. X-HyperFleet-Identity)")
+	pfs.StringVar(&identityValue, "identity-value", "", "Caller identity value sent in the identity header")
 
 	// Flags are bound in subcommand run() after config loading (osde2e pattern)
 
 	root.AddCommand(test.Cmd)
 }
 
 var (
-	configFile string
-	apiURL     string
-	logLevel   string
-	logFormat  string
-	logOutput  string
+	configFile     string
+	apiURL         string
+	logLevel       string
+	logFormat      string
+	logOutput      string
+	identityHeader string
+	identityValue  string
 )
 
 func main() {
diff --git a/cmd/hyperfleet-e2e/test/cmd.go b/cmd/hyperfleet-e2e/test/cmd.go
@@ -57,12 +57,14 @@ func run(cmd *cobra.Command, argv []string) {
 	_ = viper.BindPFlag(config.Tests.GinkgoSkip, pfs.Lookup("skip"))
 	_ = viper.BindPFlag(config.Tests.JUnitReportPath, pfs.Lookup("junit-report"))
 
-	// Bind parent command flags (api-url, logging flags)
+	// Bind parent command flags (api-url, logging flags, identity)
 	parentFlags := cmd.Parent().PersistentFlags()
 	_ = viper.BindPFlag(config.API.URL, parentFlags.Lookup("api-url"))
 	_ = viper.BindPFlag(config.Log.Level, parentFlags.Lookup("log-level"))
 	_ = viper.BindPFlag(config.Log.Format, parentFlags.Lookup("log-format"))
 	_ = viper.BindPFlag(config.Log.Output, parentFlags.Lookup("log-output"))
+	_ = viper.BindPFlag(config.Identity.Header, parentFlags.Lookup("identity-header"))
+	_ = viper.BindPFlag(config.Identity.Value, parentFlags.Lookup("identity-value"))
 
 	// Bind test environment variables
 	_ = viper.BindEnv(config.Tests.GinkgoLabelFilter, "GINKGO_LABEL_FILTER")
diff --git a/configs/config.yaml b/configs/config.yaml
@@ -114,3 +114,23 @@ adapters:
   #   - API_ADAPTERS_NODEPOOL
   nodepool:
     - "np-configmap"
+
+# ============================================================================
+# Caller Identity Configuration
+# ============================================================================
+
+identity:
+  # HTTP header name for caller identity (must match API's server.identity_header)
+  # Conventional value: X-HyperFleet-Identity
+  # Can be overridden by: HYPERFLEET_IDENTITY_HEADER
+  header: ""
+
+  # Identity value sent in the header (e.g. e2e-test@hyperfleet.local)
+  # Can be overridden by: HYPERFLEET_IDENTITY_VALUE
+  value: ""
+
+  # JWT bearer token for authenticated requests
+  # When set, sent as Authorization: Bearer <token>
+  # Can be overridden by: HYPERFLEET_IDENTITY_TOKEN
+  # NOTE: Use environment variable for sensitive tokens, not this config file
+  token: ""
diff --git a/deploy-scripts/lib/api.sh b/deploy-scripts/lib/api.sh
@@ -55,6 +55,11 @@ install_api() {
         --set "service.type=${API_SERVICE_TYPE}"
     )
 
+    # Enable caller identity via HTTP header for audit attribution
+    local identity_header="${API_IDENTITY_HEADER:-X-HyperFleet-Identity}"
+    helm_cmd+=(--set "config.server.identity_header=${identity_header}")
+    log_verbose "Identity header: ${identity_header}"
+
     # Add adapter configurations (always set both, use empty if not discovered)
     # The API chart requires both config.adapters.required.cluster and config.adapters.required.nodepool to be set
     if [[ -n "${cluster_adapters}" ]]; then
diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -34,7 +34,16 @@ You should see the command help output.
 export HYPERFLEET_API_URL=https://api.hyperfleet.example.com
 ```
 
-**Step 2**: Run tests
+**Step 2**: Configure caller identity
+
+The API enforces caller identity on mutating requests (production default). Without identity configured, tests will get `401 Unauthorized`.
+
+```bash
+export HYPERFLEET_IDENTITY_HEADER=X-HyperFleet-Identity
+export HYPERFLEET_IDENTITY_VALUE=dev-user@hyperfleet.local
+```
+
+**Step 3**: Run tests
 
 ```bash
 ./bin/hyperfleet-e2e test --label-filter=tier0
diff --git a/docs/local-kind-setup.md b/docs/local-kind-setup.md
@@ -92,6 +92,9 @@ Local kind settings are at the bottom of the file:
 | `NAMESPACE` | `hyperfleet-local` | Kubernetes namespace |
 | `HYPERFLEET_API_URL` | — | API URL for tests (`http://localhost:8000`) |
 | `MAESTRO_URL` | — | Maestro URL for tests (`http://localhost:8100`) |
+| `HYPERFLEET_IDENTITY_HEADER` | — | Identity header name (`X-HyperFleet-Identity`) |
+| `HYPERFLEET_IDENTITY_VALUE` | — | Identity value sent in the header (must be email format) |
+| `API_IDENTITY_HEADER` | `X-HyperFleet-Identity` | Header name passed to API helm chart via `config.server.identity_header` |
 
 ## Troubleshooting
 
diff --git a/e2e/cluster/creation.go b/e2e/cluster/creation.go
@@ -31,6 +31,10 @@ var _ = ginkgo.Describe("[Suite: cluster][baseline] Cluster Resource Type Lifecy
 			clusterName = cluster.Name
 			ginkgo.GinkgoWriter.Printf("Created cluster ID: %s, Name: %s\n", clusterID, clusterName)
 
+			if expected := h.ExpectedIdentity(); expected != "" {
+				Expect(cluster).To(helper.HaveAuditIdentity(expected))
+			}
+
 			ginkgo.DeferCleanup(func(ctx context.Context) {
 				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
 					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
diff --git a/e2e/cluster/delete.go b/e2e/cluster/delete.go
@@ -53,6 +53,11 @@ var _ = ginkgo.Describe("[Suite: cluster][delete] Cluster Deletion Lifecycle",
 			Expect(deletedCluster.DeletedTime).NotTo(BeNil(), "soft-deleted cluster should have deleted_time set")
 			Expect(deletedCluster.Generation).To(Equal(clusterBefore.Generation+1), "generation should increment after soft-delete")
 
+			if expected := h.ExpectedIdentity(); expected != "" {
+				Expect(deletedCluster.DeletedBy).NotTo(BeNil(), "deleted_by should be set on soft-delete")
+				Expect(string(*deletedCluster.DeletedBy)).To(Equal(expected), "deleted_by should match configured identity")
+			}
+
 			ginkgo.By("waiting for cluster adapters to finalize and cluster to be hard-deleted")
 			// Hard-delete executes atomically within the POST /adapter_statuses request that
 			// computes Reconciled=True, so there is no observable window to see Finalized=True
diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -20,12 +20,13 @@ type HyperFleetClient struct {
 }
 
 // NewHyperFleetClient creates a new HyperFleet API client.
-func NewHyperFleetClient(baseURL string, httpClient *http.Client) (*HyperFleetClient, error) {
+func NewHyperFleetClient(baseURL string, httpClient *http.Client, opts ...openapi.ClientOption) (*HyperFleetClient, error) {
 	if httpClient == nil {
 		httpClient = &http.Client{Timeout: 30 * time.Second}
 	}
 
-	client, err := openapi.NewClient(baseURL, openapi.WithHTTPClient(httpClient))
+	clientOpts := append([]openapi.ClientOption{openapi.WithHTTPClient(httpClient)}, opts...)
+	client, err := openapi.NewClient(baseURL, clientOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create client: %w", err)
 	}
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -71,6 +71,25 @@ var Tests = struct {
 	JUnitReportPath:   "tests.junitReportPath",
 }
 
+// Identity config keys
+var Identity = struct {
+	// Header is the HTTP header name for caller identity
+	// Env: HYPERFLEET_IDENTITY_HEADER
+	Header string
+
+	// Value is the identity header value
+	// Env: HYPERFLEET_IDENTITY_VALUE
+	Value string
+
+	// Token is the JWT bearer token
+	// Env: HYPERFLEET_IDENTITY_TOKEN
+	Token string
+}{
+	Header: "identity.header",
+	Value:  "identity.value",
+	Token:  "identity.token",
+}
+
 // Log config keys
 var Log = struct {
 	// Level is the minimum log level
@@ -114,6 +133,14 @@ type APIDeploymentConfig struct {
 	ChartPath string `yaml:"chartPath" mapstructure:"chartPath"`
 }
 
+// IdentityConfig contains caller identity configuration for audit attribution.
+// When configured, identity headers are injected into all API requests.
+type IdentityConfig struct {
+	Header string `yaml:"header" mapstructure:"header"` // HTTP header name (e.g. X-HyperFleet-Identity)
+	Value  string `yaml:"value" mapstructure:"value"`   // Header value (e.g. e2e-test@hyperfleet.local)
+	Token  string `yaml:"token" mapstructure:"token"`   // JWT bearer token
+}
+
 // Config represents the e2e test configuration
 type Config struct {
 	Namespace         string                  `yaml:"namespace" mapstructure:"namespace"`
@@ -127,6 +154,7 @@ type Config struct {
 	Adapters          AdaptersConfig          `yaml:"adapters" mapstructure:"adapters"`
 	AdapterDeployment AdapterDeploymentConfig `yaml:"adapterDeployment" mapstructure:"adapterDeployment"`
 	APIDeployment     APIDeploymentConfig     `yaml:"apiDeployment" mapstructure:"apiDeployment"`
+	Identity          IdentityConfig          `yaml:"identity" mapstructure:"identity"`
 }
 
 // APIConfig contains API-related configuration
@@ -431,6 +459,11 @@ func (c *Config) Validate() error {
 		return fmt.Errorf("configuration validation failed: polling.interval must be a positive duration, got %v", c.Polling.Interval)
 	}
 
+	// Validate identity config: header and value must both be set or both be empty
+	if (c.Identity.Header != "") != (c.Identity.Value != "") {
+		return fmt.Errorf("configuration validation failed: identity.header and identity.value must both be set or both be empty (got header=%q, value=%q)", c.Identity.Header, c.Identity.Value)
+	}
+
 	return nil
 }
 
@@ -461,6 +494,9 @@ func (c *Config) Display() {
 		"api_chart_repo", redactURL(c.APIDeployment.ChartRepo),
 		"api_chart_ref", valueOrNotSet(c.APIDeployment.ChartRef),
 		"api_chart_path", valueOrNotSet(c.APIDeployment.ChartPath),
+		"identity_header", valueOrNotSet(c.Identity.Header),
+		"identity_value", redactToken(c.Identity.Value),
+		"identity_token", redactToken(c.Identity.Token),
 	)
 }
 
@@ -472,6 +508,14 @@ func valueOrNotSet(value string) string {
 	return value
 }
 
+// redactToken returns RedactedPlaceholder if the token is non-empty, otherwise NotSetPlaceholder
+func redactToken(token string) string {
+	if token == "" {
+		return NotSetPlaceholder
+	}
+	return RedactedPlaceholder
+}
+
 // redactURL redacts credentials from URLs
 func redactURL(rawURL string) string {
 	if rawURL == "" {
diff --git a/pkg/helper/helper.go b/pkg/helper/helper.go
@@ -164,6 +164,13 @@ func (h *Helper) CleanupTestChannel(ctx context.Context, channelID string) error
 	return deleteErr
 }
 
+// ExpectedIdentity returns the configured caller identity value.
+// Returns empty string if no identity is configured, signalling callers to skip
+// audit assertions.
+func (h *Helper) ExpectedIdentity() string {
+	return h.Cfg.Identity.Value
+}
+
 // GetMaestroClient returns the Maestro client, initializing it lazily on first access
 // This avoids the overhead of K8s service discovery for test suites that don't use Maestro
 func (h *Helper) GetMaestroClient() *maestro.Client {
diff --git a/pkg/helper/matchers.go b/pkg/helper/matchers.go
@@ -157,6 +157,53 @@ func (m *allAdaptersGenerationMatcher) NegatedFailureMessage(_ any) string {
 	return fmt.Sprintf("expected adapters NOT at generation %d", m.generation)
 }
 
+// HaveAuditIdentity matches a *Cluster or *Resource whose CreatedBy field equals the expected identity.
+func HaveAuditIdentity(expected string) types.GomegaMatcher {
+	return &auditIdentityMatcher{expected: expected}
+}
+
+type auditIdentityMatcher struct {
+	expected string
+	actual   string
+}
+
+func (m *auditIdentityMatcher) Match(actual any) (bool, error) {
+	identity, err := extractCreatedBy(actual)
+	if err != nil {
+		return false, err
+	}
+	m.actual = identity
+	return identity == m.expected, nil
+}
+
+func (m *auditIdentityMatcher) FailureMessage(_ any) string {
+	return fmt.Sprintf("expected created_by=%q but got %q", m.expected, m.actual)
+}
+
+func (m *auditIdentityMatcher) NegatedFailureMessage(_ any) string {
+	return fmt.Sprintf("expected created_by NOT to be %q", m.expected)
+}
+
+func extractCreatedBy(actual any) (string, error) {
+	switch v := actual.(type) {
+	case *openapi.Cluster:
+		if v == nil {
+			return "", fmt.Errorf("HaveAuditIdentity expects non-nil *Cluster")
+		}
+		return string(v.CreatedBy), nil
+	case *client.Resource:
+		if v == nil {
+			return "", fmt.Errorf("HaveAuditIdentity expects non-nil *Resource")
+		}
+		if v.CreatedBy == nil {
+			return "", nil
+		}
+		return *v.CreatedBy, nil
+	default:
+		return "", fmt.Errorf("HaveAuditIdentity expects *Cluster or *Resource, got %T", actual)
+	}
+}
+
 func hasAdapterCond(conditions []openapi.AdapterCondition, condType string, status openapi.AdapterConditionStatus) bool {
 	for _, c := range conditions {
 		if c.Type == condType && c.Status == status {
diff --git a/pkg/helper/suite.go b/pkg/helper/suite.go

Original file line number	Diff line number	Diff line change
`@@ -20,12 +20,13 @@ type HyperFleetClient struct {`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`// NewHyperFleetClient creates a new HyperFleet API client.`
`23`		`-func NewHyperFleetClient(baseURL string, httpClient http.Client) (HyperFleetClient, error) {`
	`23`	`+func NewHyperFleetClient(baseURL string, httpClient http.Client, opts ...openapi.ClientOption) (HyperFleetClient, error) {`
`24`	`24`	`if httpClient == nil {`
`25`	`25`	`httpClient = &http.Client{Timeout: 30 * time.Second}`
`26`	`26`	`}`
`27`	`27`
`28`		`- client, err := openapi.NewClient(baseURL, openapi.WithHTTPClient(httpClient))`
	`28`	`+ clientOpts := append([]openapi.ClientOption{openapi.WithHTTPClient(httpClient)}, opts...)`
	`29`	`+ client, err := openapi.NewClient(baseURL, clientOpts...)`
`29`	`30`	`if err != nil {`
`30`	`31`	`return nil, fmt.Errorf("failed to create client: %w", err)`
`31`	`32`	`}`