coordinator: support insecure manifests behind opt-in

Author: msanft

coordinator/internal/stateguard/credentials.go

@@ -15,6 +15,7 @@ import (
 	"github.com/edgelesssys/contrast/internal/atls"
 	"github.com/edgelesssys/contrast/internal/attestation"
 	"github.com/edgelesssys/contrast/internal/attestation/certcache"
+	"github.com/edgelesssys/contrast/internal/attestation/insecure"
 	"github.com/edgelesssys/contrast/internal/attestation/snp"
 	"github.com/edgelesssys/contrast/internal/attestation/tdx"
 	"github.com/edgelesssys/contrast/internal/constants"
@@ -96,6 +97,12 @@ func (c *Credentials) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.A
 			logger.NewWithAttrs(logger.NewNamed(c.logger, "validator"), map[string]string{"reference-values": name}), &authInfo, name))
 	}
 
+	if state.Manifest().AllowInsecure() {
+		validators = append(validators, insecure.NewValidatorWithReportSetter(
+			logger.NewWithAttrs(logger.NewNamed(c.logger, "validator"), map[string]string{"reference-values": "insecure"}),
+			&authInfo, "insecure"))
+	}
+
 	serverCfg, err := atls.CreateAttestationServerTLSConfig(c.issuer, validators, c.attestationFailuresCounter)
 	if err != nil {
 		log.Error("Could not create TLS config", "error", err)

coordinator/internal/stateguard/stateguard.go

@@ -52,6 +52,10 @@ var (
 	// ErrConcurrentUpdate is returned by state-modifying operations if the input oldState is not
 	// the current state. This usually happens when a concurrent operation succeeded.
 	ErrConcurrentUpdate = errors.New("coordinator state was updated concurrently")
+
+	// ErrInsecureNotAllowed is returned when a manifest contains insecure platforms but the
+	// coordinator was not started with the allow-insecure flag.
+	ErrInsecureNotAllowed = errors.New("manifest contains insecure platforms, but the coordinator is not configured to allow them")
 )
 
 // Guard manages the manifest state of Contrast.
@@ -65,6 +69,9 @@ type Guard struct {
 	logger  *slog.Logger
 	metrics metrics
 
+	// allowInsecure controls whether manifests with insecure platforms are accepted.
+	allowInsecure bool
+
 	clock clock.Clock
 }
 
@@ -73,7 +80,10 @@ type metrics struct {
 }
 
 // New creates a new state Guard instance.
-func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger) *Guard {
+//
+// If allowInsecure is true, the Guard will accept manifests that contain insecure platforms.
+// Otherwise, setting such a manifest will be rejected with ErrInsecureNotAllowed.
+func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger, allowInsecure bool) *Guard {
 	manifestGeneration := promauto.With(reg).NewGauge(prometheus.GaugeOpts{
 		Subsystem: "contrast_coordinator",
 		Name:      "manifest_generation",
@@ -82,8 +92,9 @@ func New(hist *history.History, reg *prometheus.Registry, log *slog.Logger) *Gua
 	manifestGeneration.Set(0)
 
 	return &Guard{
-		hist:   hist,
-		logger: log.WithGroup("stateguard"),
+		hist:          hist,
+		logger:        log.WithGroup("stateguard"),
+		allowInsecure: allowInsecure,
 		metrics: metrics{
 			manifestGeneration: manifestGeneration,
 		},
@@ -271,6 +282,9 @@ func (g *Guard) UpdateState(_ context.Context, oldState *State, se *seedengine.S
 	if err := json.Unmarshal(manifestBytes, &mnfst); err != nil {
 		return nil, fmt.Errorf("unmarshaling manifest: %w", err)
 	}
+	if !g.allowInsecure && mnfst.AllowInsecure() {
+		return nil, ErrInsecureNotAllowed
+	}
 	policyMap := make(map[[history.HashSize]byte][]byte)
 	for _, policy := range policies {
 		policyHash, err := g.hist.SetPolicy(policy)

coordinator/internal/stateguard/stateguard_test.go

@@ -192,6 +192,37 @@ func TestResetState(t *testing.T) {
 	require.ErrorIs(err, assert.AnError)
 }
 
+func TestUpdateStateInsecure(t *testing.T) {
+	ctx := t.Context()
+
+	_, insecureManifestBytes, policies := newInsecureManifest(t)
+	se := newSeedEngine(t)
+
+	t.Run("rejected when allowInsecure is false", func(t *testing.T) {
+		require := require.New(t)
+
+		store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()})
+		hist := history.NewWithStore(slog.Default(), store)
+		g := New(hist, prometheus.NewRegistry(), slog.Default(), false)
+
+		state, err := g.UpdateState(ctx, nil, se, insecureManifestBytes, policies)
+		require.ErrorIs(err, ErrInsecureNotAllowed)
+		require.Nil(state)
+	})
+
+	t.Run("accepted when allowInsecure is true", func(t *testing.T) {
+		require := require.New(t)
+
+		store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()})
+		hist := history.NewWithStore(slog.Default(), store)
+		g := New(hist, prometheus.NewRegistry(), slog.Default(), true)
+
+		state, err := g.UpdateState(ctx, nil, se, insecureManifestBytes, policies)
+		require.NoError(err)
+		require.NotNil(state)
+	})
+}
+
 func TestConcurrentUpdateState(t *testing.T) {
 	ctx := t.Context()
 	assert := assert.New(t)
@@ -200,7 +231,7 @@ func TestConcurrentUpdateState(t *testing.T) {
 		Store: aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()}),
 	}
 	hist := history.NewWithStore(slog.Default(), store)
-	guard := New(hist, prometheus.NewRegistry(), slog.Default())
+	guard := New(hist, prometheus.NewRegistry(), slog.Default(), false)
 
 	numWorkers := 20
 
@@ -303,7 +334,7 @@ func TestWatchHistory(t *testing.T) {
 				notifications: make(chan []byte),
 			}
 			hist := history.NewWithStore(slog.Default(), store)
-			g := New(hist, prometheus.NewRegistry(), slog.Default())
+			g := New(hist, prometheus.NewRegistry(), slog.Default(), false)
 
 			_, manifestBytes, policies := newManifest(t)
 
@@ -352,7 +383,7 @@ func TestWatchHistoryLateNotifications(t *testing.T) {
 		notifications: make(chan []byte),
 	}
 	hist := history.NewWithStore(slog.Default(), store)
-	g := New(hist, prometheus.NewRegistry(), slog.Default())
+	g := New(hist, prometheus.NewRegistry(), slog.Default(), false)
 
 	_, manifestBytes, policies := newManifest(t)
 
@@ -409,7 +440,7 @@ func TestBadStoreWatcherIsRestarted(t *testing.T) {
 	store.storeUpdates.Store(&ch)
 	hist := history.NewWithStore(slog.Default(), store)
 	reg := prometheus.NewRegistry()
-	a := New(hist, reg, slog.Default())
+	a := New(hist, reg, slog.Default(), false)
 	clock := &waitingClock{
 		FakeClock:  testingclock.NewFakeClock(time.Now()),
 		afterCalls: make(chan struct{}, 1),
@@ -502,7 +533,7 @@ func newTestGuard(t *testing.T) (*Guard, *prometheus.Registry) {
 	store := aferostore.New(&afero.Afero{Fs: afero.NewMemMapFs()})
 	hist := history.NewWithStore(slog.Default(), store)
 	reg := prometheus.NewRegistry()
-	return New(hist, reg, slog.Default()), reg
+	return New(hist, reg, slog.Default(), false), reg
 }
 
 func newManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) {
@@ -542,6 +573,28 @@ func newManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) {
 	return mnfst, mnfstBytes, [][]byte{policy}
 }
 
+func newInsecureManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) {
+	t.Helper()
+	policy := []byte("=== SOME REGO HERE ===")
+	policyHash := sha256.Sum256(policy)
+	policyHashHex := manifest.NewHexString(policyHash[:])
+
+	mnfst := &manifest.Manifest{}
+	mnfst.Policies = map[manifest.HexString]manifest.PolicyEntry{
+		policyHashHex: {
+			SANs:             []string{"test"},
+			WorkloadSecretID: "test2",
+			Role:             manifest.RoleCoordinator,
+		},
+	}
+	mnfst.ReferenceValues.SNP = []manifest.SNPReferenceValues{
+		{Platform: "Metal-QEMU-Insecure"},
+	}
+	mnfstBytes, err := json.Marshal(mnfst)
+	require.NoError(t, err)
+	return mnfst, mnfstBytes, [][]byte{policy}
+}
+
 func newSeedEngine(t *testing.T) *seedengine.SeedEngine {
 	t.Helper()
 	data := make([]byte, 32)

coordinator/internal/userapi/userapi.go

@@ -151,8 +151,11 @@ func (s *Server) SetManifest(ctx context.Context, req *userapi.SetManifestReques
 	state, err := s.guard.UpdateState(ctx, oldState, se, req.GetManifest(), req.GetPolicies())
 	if err != nil {
 		code := codes.Internal
-		if errors.Is(err, stateguard.ErrConcurrentUpdate) {
+		switch {
+		case errors.Is(err, stateguard.ErrConcurrentUpdate):
 			code = codes.FailedPrecondition
+		case errors.Is(err, stateguard.ErrInsecureNotAllowed):
+			code = codes.InvalidArgument
 		}
 		return nil, status.Errorf(code, "updating Coordinator state: %v", err)
 	}

coordinator/internal/userapi/userapi_test.go

@@ -232,6 +232,34 @@ func TestSetManifest(t *testing.T) {
 		require.Equal(codes.InvalidArgument, status.Code(err))
 	})
 
+	t.Run("insecure manifest rejected", func(t *testing.T) {
+		require := require.New(t)
+
+		// Default coordinator does not allow insecure manifests.
+		coordinator := newCoordinator()
+		m := newInsecureManifest(t)
+		manifestBytes, err := json.Marshal(m)
+		require.NoError(err)
+		req := &userapi.SetManifestRequest{Manifest: manifestBytes}
+		_, err = coordinator.SetManifest(t.Context(), req)
+		require.Error(err)
+		require.Equal(codes.InvalidArgument, status.Code(err))
+		require.ErrorContains(err, "insecure")
+	})
+
+	t.Run("insecure manifest accepted when allowed", func(t *testing.T) {
+		require := require.New(t)
+
+		coordinator := newCoordinatorAllowInsecure()
+		m := newInsecureManifest(t)
+		manifestBytes, err := json.Marshal(m)
+		require.NoError(err)
+		req := &userapi.SetManifestRequest{Manifest: manifestBytes}
+		resp, err := coordinator.SetManifest(t.Context(), req)
+		require.NoError(err)
+		require.NotNil(resp)
+	})
+
 	t.Run("atomic manifest update", func(t *testing.T) {
 		require := require.New(t)
 
@@ -404,7 +432,7 @@ func TestRecovery(t *testing.T) {
 			fs := afero.NewMemMapFs()
 			store := aferostore.New(&afero.Afero{Fs: fs})
 			hist := history.NewWithStore(slog.Default(), store)
-			auth := stateguard.New(hist, prometheus.NewRegistry(), logger)
+			auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false)
 			discovery := &stubDiscovery{
 				peers: tc.peers,
 				err:   tc.peersErr,
@@ -438,7 +466,7 @@ func TestRecovery(t *testing.T) {
 			}
 
 			// Simulate a restarted Coordinator.
-			a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default())
+			a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default(), false)
 			_, err = a.GetManifests(t.Context(), nil)
 			require.ErrorContains(err, ErrNeedsRecovery.Error())
 			_, err = a.Recover(rpcContext(t.Context(), seedShareOwnerKey), recoverReq)
@@ -460,7 +488,7 @@ func TestRecoveryFlow(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	store := aferostore.New(&afero.Afero{Fs: fs})
 	hist := history.NewWithStore(slog.Default(), store)
-	auth := stateguard.New(hist, prometheus.NewRegistry(), logger)
+	auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false)
 	a := New(logger, auth, &stubDiscovery{})
 
 	// 2. A manifest is set and the returned seed is recorded.
@@ -496,7 +524,7 @@ func TestRecoveryFlow(t *testing.T) {
 	// 3. A new Coordinator is created with the existing history.
 	// GetManifests and SetManifest are expected to fail.
 
-	a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default())
+	a.guard = stateguard.New(hist, prometheus.NewRegistry(), slog.Default(), false)
 	_, err = a.SetManifest(t.Context(), req)
 	require.ErrorContains(err, ErrNeedsRecovery.Error())
 
@@ -539,7 +567,7 @@ func TestUserAPIConcurrent(t *testing.T) {
 	fs := afero.NewBasePathFs(afero.NewOsFs(), t.TempDir())
 	store := aferostore.New(&afero.Afero{Fs: fs})
 	hist := history.NewWithStore(slog.Default(), store)
-	auth := stateguard.New(hist, prometheus.NewRegistry(), logger)
+	auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false)
 	coordinator := New(logger, auth, &stubDiscovery{})
 
 	setReq := &userapi.SetManifestRequest{
@@ -853,14 +881,32 @@ func newCoordinatorWithRegistry(reg *prometheus.Registry) *Server {
 	fs := afero.NewMemMapFs()
 	store := aferostore.New(&afero.Afero{Fs: fs})
 	hist := history.NewWithStore(slog.Default(), store)
-	auth := stateguard.New(hist, reg, logger)
+	auth := stateguard.New(hist, reg, logger, false)
 	return New(logger, auth, &stubDiscovery{})
 }
 
+func newCoordinatorAllowInsecure() *Server {
+	logger := slog.Default()
+	fs := afero.NewMemMapFs()
+	store := aferostore.New(&afero.Afero{Fs: fs})
+	hist := history.NewWithStore(slog.Default(), store)
+	auth := stateguard.New(hist, prometheus.NewRegistry(), logger, true)
+	return New(logger, auth, &stubDiscovery{})
+}
+
+func newInsecureManifest(t *testing.T) *manifest.Manifest {
+	t.Helper()
+	mnfst := &manifest.Manifest{}
+	mnfst.ReferenceValues.SNP = []manifest.SNPReferenceValues{
+		{Platform: "Metal-QEMU-Insecure"},
+	}
+	return mnfst
+}
+
 func newCoordinatorWithWatcher(t *testing.T, hist *history.History) *Server {
 	t.Helper()
 	logger := slog.Default()
-	auth := stateguard.New(hist, prometheus.NewRegistry(), logger)
+	auth := stateguard.New(hist, prometheus.NewRegistry(), logger, false)
 	coordinator := New(logger, auth, &stubDiscovery{})
 
 	ctx, cancel := context.WithCancel(t.Context())

coordinator/main.go

@@ -52,6 +52,7 @@ import (
 
 const (
 	metricsEnvVar       = "CONTRAST_METRICS"
+	allowInsecureEnvVar = "CONTRAST_ALLOW_INSECURE"
 	probeAndMetricsPort = 9102
 	// transitEngineAPIPort specifies the default port to expose the transit engine API.
 	transitEngineAPIPort = "8200"
@@ -115,7 +116,12 @@ func run() (retErr error) {
 
 	hist := history.NewWithStore(logger.WithGroup("history"), store)
 
-	meshAuth := stateguard.New(hist, promRegistry, logger)
+	_, allowInsecure := os.LookupEnv(allowInsecureEnvVar)
+	if allowInsecure {
+		logger.Warn("Coordinator is configured to allow insecure manifests")
+	}
+
+	meshAuth := stateguard.New(hist, promRegistry, logger, allowInsecure)
 
 	issuer, err := issuer.New(logger)
 	if err != nil {

internal/manifest/manifest.go

@@ -119,6 +119,21 @@ func (m *Manifest) CoordinatorPolicyHash() (HexString, error) {
 	return "", errors.New("no coordinator found in manifest")
 }
 
+// AllowInsecure returns true if the manifest contains reference values for insecure platforms.
+func (m *Manifest) AllowInsecure() bool {
+	for _, v := range m.ReferenceValues.SNP {
+		if p, err := platforms.FromString(v.Platform); err == nil && platforms.IsInsecure(p) {
+			return true
+		}
+	}
+	for _, v := range m.ReferenceValues.TDX {
+		if p, err := platforms.FromString(v.Platform); err == nil && platforms.IsInsecure(p) {
+			return true
+		}
+	}
+	return false
+}
+
 // SNPValidateOpts returns validate options generators populated with the manifest's
 // SNP reference values and trusted measurement for the given runtime.
 func (m *Manifest) SNPValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]SNPValidatorOptions, error) {
@@ -128,6 +143,9 @@ func (m *Manifest) SNPValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]SN
 
 	var out []SNPValidatorOptions
 	for _, refVal := range m.ReferenceValues.SNP {
+		if p, err := platforms.FromString(refVal.Platform); err == nil && platforms.IsInsecure(p) {
+			continue
+		}
 		if len(refVal.TrustedMeasurement) == 0 {
 			return nil, errors.New("trusted measurement cannot be empty")
 		}
@@ -213,6 +231,9 @@ func (m *Manifest) TDXValidateOpts(kdsGetter *certcache.CachedHTTPSGetter) ([]TD
 
 	var out []TDXValidatorOptions
 	for _, refVal := range m.ReferenceValues.TDX {
+		if p, err := platforms.FromString(refVal.Platform); err == nil && platforms.IsInsecure(p) {
+			continue
+		}
 		verifyOpts := tdxverify.DefaultOptions()
 
 		var err error

internal/manifest/referencevalues.go

@@ -208,6 +208,9 @@ type SNPReferenceValues struct {
 
 // Validate checks the validity of all fields in the AKS reference values.
 func (r SNPReferenceValues) Validate() error {
+	if p, err := platforms.FromString(r.Platform); err == nil && platforms.IsInsecure(p) {
+		return nil
+	}
 	var minTCBErrs []error
 	if r.MinimumTCB.BootloaderVersion == nil {
 		minTCBErrs = append(minTCBErrs, newValidationError("BootloaderVersion", ExpectedMissingReferenceValueError{Err: errors.New("field cannot be empty")}))
@@ -312,6 +315,9 @@ type TDXReferenceValues struct {
 
 // Validate checks the validity of all fields in the bare metal TDX reference values.
 func (r TDXReferenceValues) Validate() error {
+	if p, err := platforms.FromString(r.Platform); err == nil && platforms.IsInsecure(p) {
+		return nil
+	}
 	var errs []error
 	if err := validateHexString(r.MrTd, 48); err != nil {
 		errs = append(errs, newValidationError("MrTd", err))