Skip to content

Commit c8d5eea

Browse files
ivanauthivanauth
committed
Support separate datastore URIs for migrations
- Add support for migration_datastore_uri in secret - Migration jobs will use migration_datastore_uri if present - Application pods continue to use datastore_uri - Add tests for migration datastore URI functionality - Add example showing how to use separate credentials This allows using elevated database privileges for migrations while running the application with least-privilege credentials. Fixes #338
1 parent 27f03e3 commit c8d5eea

6 files changed

Lines changed: 278 additions & 15 deletions

File tree

‎examples/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# SpiceDB Operator Examples
2+
3+
This directory contains examples of how to configure and deploy SpiceDB clusters using the SpiceDB operator.
4+
5+
## Examples
6+
7+
- [cockroachdb-tls-ingress](cockroachdb-tls-ingress/) - Production-ready setup with CockroachDB, TLS, and ingress
8+
- [alternative-registry](alternative-registry/) - Using a private or alternative container registry for SpiceDB images
9+
- [separate-migration-datastore-uri](separate-migration-datastore-uri/) - Using separate database credentials for migrations vs runtime
10+
11+
## Getting Started
12+
13+
Each example includes:
14+
15+
- A README with detailed explanations
16+
- YAML manifests that can be applied to your cluster
17+
- Configuration best practices for the specific use case
18+
19+
Choose an example that matches your needs and follow the instructions in its README.

‎examples/separate-migration-datastore-uri/README.md‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Separate Migration Datastore URI
2+
3+
This example demonstrates how to use separate database connection strings for migrations versus normal SpiceDB operation.
4+
5+
## Overview
6+
7+
The SpiceDB operator supports using different database credentials for:
8+
9+
- **Migrations**: Often require elevated privileges (CREATE TABLE, DROP TABLE, ALTER TABLE)
10+
- **Application Runtime**: Should use least-privilege credentials (SELECT, INSERT, UPDATE, DELETE)
11+
12+
This separation follows security best practices by ensuring the SpiceDB application pods don't have unnecessary database permissions.
13+
14+
## How it Works
15+
16+
When the operator detects a `migration_datastore_uri` key in the secret, it will:
17+
18+
1. Use `migration_datastore_uri` for migration jobs
19+
2. Continue using `datastore_uri` for the SpiceDB application pods
20+
21+
## Example Configuration
22+
23+
See [spicedb-cluster.yaml](spicedb-cluster.yaml) for the complete example.
24+
25+
The key part is the secret configuration:
26+
27+
```yaml
28+
apiVersion: v1
29+
kind: Secret
30+
metadata:
31+
name: spicedb-config
32+
stringData:
33+
# Used by SpiceDB application pods - limited privileges
34+
datastore_uri: "postgresql://spicedb_user:password@postgres:5432/spicedb?sslmode=require"
35+
36+
# Used by migration jobs - elevated privileges
37+
migration_datastore_uri: "postgresql://spicedb_admin:admin_password@postgres:5432/spicedb?sslmode=require"
38+
39+
preshared_key: "your-secure-preshared-key"
40+
```
41+
42+
## Database User Setup
43+
44+
Here's an example of how to set up the PostgreSQL users:
45+
46+
```sql
47+
-- Create the admin user (for migrations)
48+
CREATE USER spicedb_admin WITH PASSWORD 'admin_password';
49+
GRANT CREATE ON DATABASE spicedb TO spicedb_admin;
50+
GRANT ALL PRIVILEGES ON SCHEMA public TO spicedb_admin;
51+
52+
-- Create the application user (for runtime)
53+
CREATE USER spicedb_user WITH PASSWORD 'password';
54+
55+
-- After migrations are run, grant appropriate permissions
56+
-- The migration will create the tables, then you can run:
57+
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO spicedb_user;
58+
GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO spicedb_user;
59+
```
60+
61+
## Security Considerations
62+
63+
- Store credentials in a proper secret management system
64+
- Use strong, unique passwords for both users
65+
- Consider using SSL/TLS for database connections
66+
- Regularly rotate credentials
67+
- Monitor database access logs

‎examples/separate-migration-datastore-uri/spicedb-cluster.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: spicedb-separate-migration
5+
---
6+
apiVersion: authzed.com/v1alpha1
7+
kind: SpiceDBCluster
8+
metadata:
9+
name: example-separate-migration
10+
namespace: spicedb-separate-migration
11+
spec:
12+
# Number of SpiceDB replicas
13+
replicas: 3
14+
15+
config:
16+
datastoreEngine: postgres
17+
logLevel: info
18+
19+
secretName: spicedb-config
20+
---
21+
apiVersion: v1
22+
kind: Secret
23+
metadata:
24+
name: spicedb-config
25+
namespace: spicedb-separate-migration
26+
stringData:
27+
# Application datastore URI - used by SpiceDB pods
28+
# This user should have limited privileges (SELECT, INSERT, UPDATE, DELETE)
29+
datastore_uri: "postgresql://CHANGE-ME-APP-USER:CHANGE-ME-APP-PASSWORD@CHANGE-ME-POSTGRES-HOST:5432/CHANGE-ME-DATABASE?sslmode=require"
30+
31+
# Migration datastore URI - used only by migration jobs
32+
# This user needs elevated privileges (CREATE, DROP, ALTER tables)
33+
migration_datastore_uri: "postgresql://CHANGE-ME-ADMIN-USER:CHANGE-ME-ADMIN-PASSWORD@CHANGE-ME-POSTGRES-HOST:5432/CHANGE-ME-DATABASE?sslmode=require"
34+
35+
# Shared preshared key for API authentication
36+
preshared_key: "CHANGE-ME-TO-A-VERY-SECRET-PRESHARED-KEY"

‎pkg/config/config.go‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -136,17 +136,18 @@ type Config struct {
136136
// MigrationConfig stores data that is relevant for running migrations
137137
// or deciding if migrations need to be run
138138
type MigrationConfig struct {
139-
TargetMigration string
140-
TargetPhase string
141-
MigrationLogLevel string
142-
DatastoreEngine string
143-
DatastoreURI string
144-
SpannerCredsSecretRef string
145-
TargetSpiceDBImage string
146-
EnvPrefix string
147-
SpiceDBCmd string
148-
DatastoreTLSSecretName string
149-
SpiceDBVersion *v1alpha1.SpiceDBVersion
139+
TargetMigration string
140+
TargetPhase string
141+
MigrationLogLevel string
142+
DatastoreEngine string
143+
DatastoreURI string
144+
HasMigrationDatastoreURI bool
145+
SpannerCredsSecretRef string
146+
TargetSpiceDBImage string
147+
EnvPrefix string
148+
SpiceDBCmd string
149+
DatastoreTLSSecretName string
150+
SpiceDBVersion *v1alpha1.SpiceDBVersion
150151
}
151152

152153
// SpiceConfig contains config relevant to running spicedb or determining
@@ -286,6 +287,14 @@ func NewConfig(cluster *v1alpha1.SpiceDBCluster, globalConfig *OperatorConfig, s
286287
errs = append(errs, fmt.Errorf("secret must contain a datastore_uri field"))
287288
}
288289
migrationConfig.DatastoreURI = string(datastoreURI)
290+
291+
// Check for separate migration datastore URI
292+
if migrationURI, ok := secret.Data["migration_datastore_uri"]; ok && len(migrationURI) > 0 {
293+
migrationConfig.HasMigrationDatastoreURI = true
294+
// Note: We don't store the actual URI in the config for security reasons
295+
// It will be read from the secret by the migration job
296+
}
297+
289298
psk, ok = secret.Data["preshared_key"]
290299
if !ok {
291300
errs = append(errs, fmt.Errorf("secret must contain a preshared_key field"))
@@ -666,9 +675,16 @@ func (c *Config) jobName(migrationHash string) string {
666675

667676
func (c *Config) unpatchedMigrationJob(migrationHash string) *applybatchv1.JobApplyConfiguration {
668677
envPrefix := c.SpiceConfig.EnvPrefix
678+
679+
// Use migration_datastore_uri if present, otherwise fall back to datastore_uri
680+
datastoreURIKey := "datastore_uri"
681+
if c.HasMigrationDatastoreURI {
682+
datastoreURIKey = "migration_datastore_uri"
683+
}
684+
669685
envVars := []*applycorev1.EnvVarApplyConfiguration{
670686
applycorev1.EnvVar().WithName(envPrefix + "_LOG_LEVEL").WithValue(c.MigrationLogLevel),
671-
applycorev1.EnvVar().WithName(envPrefix + "_DATASTORE_CONN_URI").WithValueFrom(applycorev1.EnvVarSource().WithSecretKeyRef(applycorev1.SecretKeySelector().WithName(c.SecretName).WithKey("datastore_uri"))),
687+
applycorev1.EnvVar().WithName(envPrefix + "_DATASTORE_CONN_URI").WithValueFrom(applycorev1.EnvVarSource().WithSecretKeyRef(applycorev1.SecretKeySelector().WithName(c.SecretName).WithKey(datastoreURIKey))),
672688
applycorev1.EnvVar().WithName(envPrefix + "_SECRETS").WithValueFrom(applycorev1.EnvVarSource().WithSecretKeyRef(applycorev1.SecretKeySelector().WithName(c.SecretName).WithKey("migration_secrets").WithOptional(true))),
673689
}
674690

‎pkg/config/config_test.go‎

Lines changed: 126 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1388,6 +1388,91 @@ func TestNewConfig(t *testing.T) {
13881388
},
13891389
wantPortCount: 4,
13901390
},
1391+
{
1392+
name: "separate migration datastore URI",
1393+
args: args{
1394+
cluster: v1alpha1.ClusterSpec{Config: json.RawMessage(`
1395+
{
1396+
"logLevel": "info",
1397+
"datastoreEngine": "cockroachdb"
1398+
}
1399+
`)},
1400+
globalConfig: OperatorConfig{
1401+
ImageName: "image",
1402+
UpdateGraph: updates.UpdateGraph{
1403+
Channels: []updates.Channel{
1404+
{
1405+
Name: "cockroachdb",
1406+
Metadata: map[string]string{"datastore": "cockroachdb", "default": "true"},
1407+
Nodes: []updates.State{
1408+
{ID: "v1", Tag: "v1"},
1409+
},
1410+
Edges: map[string][]string{"v1": {}},
1411+
},
1412+
},
1413+
},
1414+
},
1415+
secret: &corev1.Secret{Data: map[string][]byte{
1416+
"datastore_uri": []byte("uri"),
1417+
"migration_datastore_uri": []byte("migration-uri"),
1418+
"preshared_key": []byte("psk"),
1419+
}},
1420+
},
1421+
wantWarnings: []error{fmt.Errorf("no TLS configured, consider setting \"tlsSecretName\"")},
1422+
want: &Config{
1423+
MigrationConfig: MigrationConfig{
1424+
MigrationLogLevel: "debug",
1425+
DatastoreEngine: "cockroachdb",
1426+
DatastoreURI: "uri",
1427+
HasMigrationDatastoreURI: true,
1428+
SpannerCredsSecretRef: "",
1429+
TargetSpiceDBImage: "image:v1",
1430+
EnvPrefix: "SPICEDB",
1431+
SpiceDBCmd: "spicedb",
1432+
DatastoreTLSSecretName: "",
1433+
TargetMigration: "head",
1434+
SpiceDBVersion: &v1alpha1.SpiceDBVersion{
1435+
Name: "v1",
1436+
Channel: "cockroachdb",
1437+
Attributes: []v1alpha1.SpiceDBVersionAttributes{
1438+
v1alpha1.SpiceDBVersionAttributesMigration,
1439+
},
1440+
},
1441+
},
1442+
SpiceConfig: SpiceConfig{
1443+
LogLevel: "info",
1444+
SkipMigrations: false,
1445+
Name: "test",
1446+
Namespace: "test",
1447+
UID: "1",
1448+
Replicas: 2,
1449+
PresharedKey: "psk",
1450+
EnvPrefix: "SPICEDB",
1451+
SpiceDBCmd: "spicedb",
1452+
ServiceAccountName: "test",
1453+
DispatchEnabled: true,
1454+
DispatchUpstreamCASecretPath: "tls.crt",
1455+
ProjectLabels: true,
1456+
ProjectAnnotations: true,
1457+
Passthrough: map[string]string{
1458+
"datastoreEngine": "cockroachdb",
1459+
"dispatchClusterEnabled": "true",
1460+
"terminationLogPath": "/dev/termination-log",
1461+
},
1462+
},
1463+
},
1464+
wantEnvs: []string{
1465+
"SPICEDB_POD_NAME=FIELD_REF=metadata.name",
1466+
"SPICEDB_LOG_LEVEL=info",
1467+
"SPICEDB_GRPC_PRESHARED_KEY=preshared_key",
1468+
"SPICEDB_DATASTORE_CONN_URI=datastore_uri",
1469+
"SPICEDB_DISPATCH_UPSTREAM_ADDR=kubernetes:///test.test:dispatch",
1470+
"SPICEDB_DATASTORE_ENGINE=cockroachdb",
1471+
"SPICEDB_DISPATCH_CLUSTER_ENABLED=true",
1472+
"SPICEDB_TERMINATION_LOG_PATH=/dev/termination-log",
1473+
},
1474+
wantPortCount: 4,
1475+
},
13911476
{
13921477
name: "disable dispatch",
13931478
args: args{
@@ -2341,13 +2426,32 @@ metadata:
23412426
},
23422427
wantJob: expectedJob(),
23432428
},
2429+
{
2430+
name: "uses migration datastore URI when present",
2431+
cluster: v1alpha1.ClusterSpec{
2432+
Config: json.RawMessage(`
2433+
{
2434+
"logLevel": "debug",
2435+
"datastoreEngine": "cockroachdb"
2436+
}
2437+
`),
2438+
},
2439+
wantJob: expectedJob(func(_ *applybatchv1.JobApplyConfiguration) {
2440+
// The test below will verify the correct env var is set
2441+
}),
2442+
},
23442443
}
23452444
for _, tt := range tests {
23462445
t.Run(tt.name, func(t *testing.T) {
23472446
secret := &corev1.Secret{Data: map[string][]byte{
23482447
"datastore_uri": []byte("uri"),
23492448
"preshared_key": []byte("psk"),
23502449
}}
2450+
2451+
// Add migration_datastore_uri for specific test
2452+
if tt.name == "uses migration datastore URI when present" {
2453+
secret.Data["migration_datastore_uri"] = []byte("migration-uri")
2454+
}
23512455
cluster := &v1alpha1.SpiceDBCluster{
23522456
ObjectMeta: metav1.ObjectMeta{
23532457
Name: "test",
@@ -2364,7 +2468,28 @@ metadata:
23642468
gotJob, err := json.Marshal(got.MigrationJob("1"))
23652469
require.NoError(t, err)
23662470

2367-
require.JSONEq(t, string(wantJob), string(gotJob))
2471+
// For migration datastore URI test, we check the env var specifically
2472+
if tt.name == "uses migration datastore URI when present" {
2473+
// Verify the migration job uses migration_datastore_uri key
2474+
job := got.MigrationJob("1")
2475+
containers := job.Spec.Template.Spec.Containers
2476+
require.Len(t, containers, 1)
2477+
2478+
var foundDatastoreURI bool
2479+
for _, env := range containers[0].Env {
2480+
if env.Name != nil && *env.Name == "SPICEDB_DATASTORE_CONN_URI" {
2481+
foundDatastoreURI = true
2482+
require.NotNil(t, env.ValueFrom)
2483+
require.NotNil(t, env.ValueFrom.SecretKeyRef)
2484+
require.NotNil(t, env.ValueFrom.SecretKeyRef.Key)
2485+
require.Equal(t, "migration_datastore_uri", *env.ValueFrom.SecretKeyRef.Key)
2486+
}
2487+
}
2488+
require.True(t, foundDatastoreURI, "SPICEDB_DATASTORE_CONN_URI env var not found")
2489+
} else {
2490+
// For other tests, use the original JSONEq assertion
2491+
require.JSONEq(t, string(wantJob), string(gotJob))
2492+
}
23682493
})
23692494
}
23702495
}

‎pkg/controller/validate_config_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ func TestValidateConfigHandler(t *testing.T) {
4545
Status: v1alpha1.ClusterStatus{
4646
Image: "image:v1",
4747
Migration: "head",
48-
TargetMigrationHash: "69066f71d9cf4a1c",
49-
CurrentMigrationHash: "69066f71d9cf4a1c",
48+
TargetMigrationHash: "d6d2c3e587329b6e",
49+
CurrentMigrationHash: "d6d2c3e587329b6e",
5050
CurrentVersion: &v1alpha1.SpiceDBVersion{
5151
Name: "v1",
5252
Channel: "cockroachdb",

0 commit comments

Comments
 (0)