Skip to content

Commit 7c70a30

Browse files
mgold1234mgold1234
committed
db: Add service_snapshots JSONB column for compliance policy tracking
Add service_snapshots JSONB column to blueprint_versions table. This enables us to store compliance policy snapshots and tracking blueprint TOML generated from policies and detecting when compliance customizations become obsolete. The stored snapshots allow us to: Compare policy changes between blueprint versions Identify customizations that were added by previous policies but are no longer required Inform users about potentially superfluous customizations Changes: Add service_snapshots JSONB column to blueprint_versions table Add ServiceSnapshots and ComplianceSnapshot structs Add InsertBlueprintWithServiceSnapshots() and UpdateBlueprintWithServiceSnapshots() functions
1 parent 3a8b4ea commit 7c70a30

4 files changed

Lines changed: 225 additions & 50 deletions

File tree

‎cmd/image-builder-db-test/main_test.go‎

Lines changed: 191 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func testInsertCompose(ctx context.Context, t *testing.T) {
4343

4444
tutils.MigrateTern(ctx, t)
4545

46-
err = d.InsertBlueprint(ctx, blueprintId, versionId, ORGID1, ANR1, "blueprint", "blueprint desc", []byte("{}"), []byte("{}"))
46+
err = d.InsertBlueprint(ctx, blueprintId, versionId, ORGID1, ANR1, "blueprint", "blueprint desc", []byte("{}"), []byte("{}"), nil)
4747
require.NoError(t, err)
4848

4949
// test
@@ -318,7 +318,7 @@ func testBlueprints(ctx context.Context, t *testing.T) {
318318

319319
id := uuid.New()
320320
versionId := uuid.New()
321-
err = d.InsertBlueprint(ctx, id, versionId, ORGID1, ANR1, name1, description1, bodyJson1, []byte("{}"))
321+
err = d.InsertBlueprint(ctx, id, versionId, ORGID1, ANR1, name1, description1, bodyJson1, []byte("{}"), nil)
322322
require.NoError(t, err)
323323

324324
entry, err := d.GetBlueprint(ctx, id, ORGID1, nil)
@@ -352,7 +352,7 @@ func testBlueprints(ctx context.Context, t *testing.T) {
352352
require.NoError(t, err)
353353

354354
newVersionId := uuid.New()
355-
err = d.UpdateBlueprint(ctx, newVersionId, id, ORGID1, name2, description2, bodyJson2)
355+
err = d.UpdateBlueprint(ctx, newVersionId, id, ORGID1, name2, description2, bodyJson2, nil)
356356
require.NoError(t, err)
357357
entryUpdated, err := d.GetBlueprint(ctx, id, ORGID1, nil)
358358
require.NoError(t, err)
@@ -390,7 +390,7 @@ func testBlueprints(ctx context.Context, t *testing.T) {
390390
bodyJson3, err := json.Marshal(body3)
391391
require.NoError(t, err)
392392
newBlueprintId := uuid.New()
393-
err = d.UpdateBlueprint(ctx, newBlueprintId, id, ORGID2, name3, description3, bodyJson3)
393+
err = d.UpdateBlueprint(ctx, newBlueprintId, id, ORGID2, name3, description3, bodyJson3, nil)
394394
require.Error(t, err)
395395
entryAfterInvalidUpdate, err := d.GetBlueprint(ctx, id, ORGID1, nil)
396396
require.NoError(t, err)
@@ -407,19 +407,19 @@ func testBlueprints(ctx context.Context, t *testing.T) {
407407
newestBlueprintName := "new name"
408408

409409
// Fail to insert blueprint with the same name
410-
err = d.InsertBlueprint(ctx, newestBlueprintId, newestBlueprintVersionId, ORGID1, ANR1, newestBlueprintName, "desc", bodyJson1, []byte("{}"))
410+
err = d.InsertBlueprint(ctx, newestBlueprintId, newestBlueprintVersionId, ORGID1, ANR1, newestBlueprintName, "desc", bodyJson1, []byte("{}"), nil)
411411
require.Error(t, err)
412412

413413
newestBlueprintName = "New name 2"
414-
err = d.InsertBlueprint(ctx, newestBlueprintId, newestBlueprintVersionId, ORGID1, ANR1, newestBlueprintName, "desc", bodyJson1, []byte("{}"))
414+
err = d.InsertBlueprint(ctx, newestBlueprintId, newestBlueprintVersionId, ORGID1, ANR1, newestBlueprintName, "desc", bodyJson1, []byte("{}"), nil)
415415
require.NoError(t, err)
416416
entries, bpCount, err := d.GetBlueprints(ctx, ORGID1, 100, 0)
417417
require.NoError(t, err)
418418
require.Equal(t, 2, bpCount)
419419
require.Equal(t, entries[0].Name, newestBlueprintName)
420420
require.Equal(t, entries[1].Version, 2)
421421

422-
err = d.InsertBlueprint(ctx, uuid.New(), uuid.New(), ORGID1, ANR1, "unique name", "unique desc", bodyJson1, []byte("{}"))
422+
err = d.InsertBlueprint(ctx, uuid.New(), uuid.New(), ORGID1, ANR1, "unique name", "unique desc", bodyJson1, []byte("{}"), nil)
423423
entries, count, err := d.FindBlueprints(ctx, ORGID1, "", 100, 0)
424424
require.NoError(t, err)
425425
require.Equal(t, 3, count)
@@ -460,7 +460,39 @@ func testGetBlueprintComposes(ctx context.Context, t *testing.T) {
460460

461461
id := uuid.New()
462462
versionId := uuid.New()
463-
err = d.InsertBlueprint(ctx, id, versionId, ORGID1, ANR1, "name", "desc", []byte("{}"), []byte("{}"))
463+
body1 := v1.BlueprintBody{
464+
Distribution: "rhel-8",
465+
ImageRequests: []v1.ImageRequest{
466+
{
467+
Architecture: "x86_64",
468+
ImageType: "guest-image",
469+
},
470+
},
471+
Customizations: v1.Customizations{
472+
Packages: common.ToPtr([]string{"vim", "git"}),
473+
},
474+
}
475+
bodyJson1, err := json.Marshal(body1)
476+
require.NoError(t, err)
477+
478+
policyCustomizations1 := &v1.Customizations{
479+
Packages: &[]string{"vim", "git", "curl", "admin"},
480+
Hostname: common.ToPtr("rhel8-server"),
481+
}
482+
483+
policyCustomizations1JSON, err := json.Marshal(policyCustomizations1)
484+
require.NoError(t, err)
485+
486+
serviceSnapshots1 := db.ServiceSnapshots{
487+
Compliance: &db.ComplianceSnapshot{
488+
PolicyId: uuid.New(),
489+
PolicyCustomizations: policyCustomizations1JSON,
490+
},
491+
}
492+
serviceSnapshotsJson1, err := json.Marshal(serviceSnapshots1)
493+
require.NoError(t, err)
494+
495+
err = d.InsertBlueprint(ctx, id, versionId, ORGID1, ANR1, "name", "desc", bodyJson1, []byte("{}"), serviceSnapshotsJson1)
464496
require.NoError(t, err)
465497

466498
// get latest version
@@ -469,41 +501,140 @@ func testGetBlueprintComposes(ctx context.Context, t *testing.T) {
469501
require.Equal(t, 1, version)
470502

471503
version2Id := uuid.New()
472-
err = d.UpdateBlueprint(ctx, version2Id, id, ORGID1, "name", "desc2", []byte("{}"))
504+
505+
body2 := v1.BlueprintBody{
506+
Distribution: "rhel-9",
507+
ImageRequests: []v1.ImageRequest{
508+
{
509+
Architecture: "x86_64",
510+
ImageType: "ami",
511+
},
512+
},
513+
Customizations: v1.Customizations{
514+
Packages: common.ToPtr([]string{"httpd", "vim", "git", "curl"}),
515+
},
516+
}
517+
bodyJson2, err := json.Marshal(body2)
473518
require.NoError(t, err)
474519

475-
clientId := "ui"
476-
err = d.InsertCompose(ctx, uuid.New(), ANR1, EMAIL1, ORGID1, common.ToPtr("image1"), []byte("{}"), &clientId, &versionId)
520+
policyCustomizations2 := &v1.Customizations{
521+
Packages: &[]string{"httpd", "vim", "git", "curl", "nginx", "firewalld", "webadmin"},
522+
Hostname: common.ToPtr("rhel9-webserver"),
523+
Services: &v1.Services{
524+
Enabled: &[]string{"httpd", "nginx", "firewalld"},
525+
},
526+
Timezone: &v1.Timezone{
527+
Timezone: common.ToPtr("UTC"),
528+
},
529+
}
530+
531+
policyCustomizations2JSON, err := json.Marshal(policyCustomizations2)
477532
require.NoError(t, err)
478-
err = d.InsertCompose(ctx, uuid.New(), ANR1, EMAIL1, ORGID1, common.ToPtr("image2"), []byte("{}"), &clientId, &versionId)
533+
534+
serviceSnapshots2 := db.ServiceSnapshots{
535+
Compliance: &db.ComplianceSnapshot{
536+
PolicyId: uuid.New(),
537+
PolicyCustomizations: policyCustomizations2JSON,
538+
},
539+
}
540+
serviceSnapshotsJson2, err := json.Marshal(serviceSnapshots2)
479541
require.NoError(t, err)
480-
err = d.InsertCompose(ctx, uuid.New(), ANR1, EMAIL1, ORGID1, common.ToPtr("image3"), []byte("{}"), &clientId, nil)
542+
543+
err = d.UpdateBlueprint(ctx, version2Id, id, ORGID1, "name", "desc2", bodyJson2, serviceSnapshotsJson2)
481544
require.NoError(t, err)
482-
err = d.InsertCompose(ctx, uuid.New(), ANR1, EMAIL1, ORGID1, common.ToPtr("image4"), []byte("{}"), &clientId, &version2Id)
545+
546+
clientId := "ui"
547+
composeRequest1 := []byte(`{
548+
"distribution": "rhel-8",
549+
"image_requests": [{
550+
"architecture": "x86_64",
551+
"image_type": "guest-image",
552+
"upload_request": {
553+
"type": "aws.s3",
554+
"options": {
555+
"region": "us-east-1"
556+
}
557+
}
558+
}],
559+
"customizations": {
560+
"packages": ["vim", "git"],
561+
"users": [{"name": "testuser", "key": "ssh-rsa AAAAB3..."}]
562+
}
563+
}`)
564+
565+
composeRequest2 := []byte(`{
566+
"distribution": "rhel-9",
567+
"image_requests": [{
568+
"architecture": "aarch64",
569+
"image_type": "edge-installer",
570+
"upload_request": {
571+
"type": "azure.storage",
572+
"options": {
573+
"resource_group": "test-rg"
574+
}
575+
}
576+
}],
577+
"customizations": {
578+
"services": {"enabled": ["httpd", "nginx"]}
579+
}
580+
}`)
581+
582+
composeRequest3 := []byte(`{
583+
"distribution": "fedora-38",
584+
"image_requests": [{
585+
"architecture": "x86_64",
586+
"image_type": "qcow2"
587+
}],
588+
"customizations": {
589+
"hostname": "test-host"
590+
}
591+
}`)
592+
593+
composeRequest4 := []byte(`{
594+
"distribution": "rhel-8",
595+
"image_requests": [{
596+
"architecture": "x86_64",
597+
"image_type": "ami",
598+
"upload_request": {
599+
"type": "aws.ec2",
600+
"options": {
601+
"region": "us-west-2",
602+
"instance_type": "t3.micro"
603+
}
604+
}
605+
}]
606+
}`)
607+
608+
compose1Id := uuid.New()
609+
err = d.InsertCompose(ctx, compose1Id, ANR1, EMAIL1, ORGID1, common.ToPtr("rhel8-guest-image"), composeRequest1, &clientId, &versionId)
483610
require.NoError(t, err)
484611

485-
count, err := d.CountBlueprintComposesSince(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), nil)
612+
compose2Id := uuid.New()
613+
err = d.InsertCompose(ctx, compose2Id, ANR1, EMAIL1, ORGID1, common.ToPtr("rhel9-edge-installer"), composeRequest2, &clientId, &versionId)
486614
require.NoError(t, err)
487-
require.Equal(t, 3, count)
488-
entries, err := d.GetBlueprintComposes(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), 10, 0, nil)
615+
616+
compose3Id := uuid.New()
617+
err = d.InsertCompose(ctx, compose3Id, ANR1, EMAIL1, ORGID1, common.ToPtr("fedora38-qcow2"), composeRequest3, &clientId, nil)
489618
require.NoError(t, err)
490-
require.Len(t, entries, 3)
491-
// retrieves fresh first
492-
require.Equal(t, "image4", *entries[0].ImageName)
493-
require.Equal(t, "image2", *entries[1].ImageName)
494-
require.Equal(t, "image1", *entries[2].ImageName)
495619

496-
composes, count, err := d.GetComposes(ctx, ORGID1, fortnight, 100, 0, []string{})
620+
compose4Id := uuid.New()
621+
err = d.InsertCompose(ctx, compose4Id, ANR1, EMAIL1, ORGID1, common.ToPtr("rhel8-ami"), composeRequest4, &clientId, &version2Id)
497622
require.NoError(t, err)
498-
require.Equal(t, id, *composes[0].BlueprintId)
499-
require.Equal(t, 2, *composes[0].BlueprintVersion)
500623

501-
count, err = d.CountBlueprintComposesSince(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), nil)
624+
count, err := d.CountBlueprintComposesSince(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), nil)
502625
require.NoError(t, err)
503626
require.Equal(t, 3, count)
504-
entries, err = d.GetBlueprintComposes(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), 10, 0, nil)
627+
entries, err := d.GetBlueprintComposes(ctx, ORGID1, id, nil, (time.Hour * 24 * 14), 10, 0, nil)
505628
require.NoError(t, err)
506629
require.Len(t, entries, 3)
630+
require.Equal(t, "rhel8-ami", *entries[0].ImageName)
631+
require.Equal(t, "rhel9-edge-installer", *entries[1].ImageName)
632+
require.Equal(t, "rhel8-guest-image", *entries[2].ImageName)
633+
634+
var requestData map[string]any
635+
err = json.Unmarshal(entries[0].Request, &requestData)
636+
require.NoError(t, err)
637+
require.Equal(t, "rhel-8", requestData["distribution"])
507638

508639
// get composes for specific version
509640
count, err = d.CountBlueprintComposesSince(ctx, ORGID1, id, common.ToPtr(2), (time.Hour * 24 * 14), nil)
@@ -512,13 +643,45 @@ func testGetBlueprintComposes(ctx context.Context, t *testing.T) {
512643
entries, err = d.GetBlueprintComposes(ctx, ORGID1, id, common.ToPtr(2), (time.Hour * 24 * 14), 10, 0, nil)
513644
require.NoError(t, err)
514645
require.Len(t, entries, 1)
515-
require.Equal(t, "image4", *entries[0].ImageName)
646+
require.Equal(t, "rhel8-ami", *entries[0].ImageName)
516647
require.Equal(t, 2, entries[0].BlueprintVersion)
517648

518649
// get latest version
519650
version, err = d.GetLatestBlueprintVersionNumber(ctx, ORGID1, id)
520651
require.NoError(t, err)
521652
require.Equal(t, 2, version)
653+
654+
blueprintEntry, err := d.GetBlueprint(ctx, id, ORGID1, common.ToPtr(2))
655+
require.NoError(t, err)
656+
retrievedBlueprint, err := v1.BlueprintFromEntry(blueprintEntry)
657+
require.NoError(t, err)
658+
require.Equal(t, v1.Distributions("rhel-9"), retrievedBlueprint.Distribution)
659+
660+
var retrievedServiceSnapshots db.ServiceSnapshots
661+
err = json.Unmarshal(blueprintEntry.ServiceSnapshots, &retrievedServiceSnapshots)
662+
require.NoError(t, err)
663+
require.NotNil(t, retrievedServiceSnapshots.Compliance)
664+
665+
var customizations map[string]any
666+
err = json.Unmarshal(retrievedServiceSnapshots.Compliance.PolicyCustomizations, &customizations)
667+
require.NoError(t, err)
668+
require.Contains(t, customizations["hostname"], "rhel9-webserver")
669+
require.Contains(t, customizations["packages"], "httpd")
670+
require.Contains(t, customizations["packages"], "webadmin")
671+
672+
blueprintEntry1, err := d.GetBlueprint(ctx, id, ORGID1, common.ToPtr(1))
673+
require.NoError(t, err)
674+
675+
var retrievedServiceSnapshots1 db.ServiceSnapshots
676+
err = json.Unmarshal(blueprintEntry1.ServiceSnapshots, &retrievedServiceSnapshots1)
677+
require.NoError(t, err)
678+
require.NotNil(t, retrievedServiceSnapshots1.Compliance)
679+
var customizations1 map[string]any
680+
err = json.Unmarshal(retrievedServiceSnapshots1.Compliance.PolicyCustomizations, &customizations1)
681+
require.NoError(t, err)
682+
require.Contains(t, customizations1["hostname"], "rhel8-server")
683+
require.Contains(t, customizations1["packages"], "admin")
684+
require.Contains(t, customizations1["packages"], "vim")
522685
}
523686

524687
func TestAll(t *testing.T) {

‎internal/db/db.go‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,14 @@ type CloneEntry struct {
4545
}
4646

4747
type BlueprintEntry struct {
48-
Id uuid.UUID
49-
VersionId uuid.UUID
50-
Version int
51-
Body json.RawMessage
52-
Name string
53-
Description string
54-
Metadata json.RawMessage
48+
Id uuid.UUID
49+
VersionId uuid.UUID
50+
Version int
51+
Body json.RawMessage
52+
Name string
53+
Description string
54+
Metadata json.RawMessage
55+
ServiceSnapshots json.RawMessage
5556
}
5657

5758
type BlueprintWithNoBody struct {
@@ -62,6 +63,15 @@ type BlueprintWithNoBody struct {
6263
LastModifiedAt time.Time
6364
}
6465

66+
type ServiceSnapshots struct {
67+
Compliance *ComplianceSnapshot `json:"compliance,omitempty"`
68+
}
69+
70+
type ComplianceSnapshot struct {
71+
PolicyId uuid.UUID `json:"policy_id"`
72+
PolicyCustomizations json.RawMessage `json:"policy_customizations"`
73+
}
74+
6575
type DB interface {
6676
InsertCompose(ctx context.Context, jobId uuid.UUID, accountNumber, email, orgId string, imageName *string, request json.RawMessage, clientId *string, blueprintVersionId *uuid.UUID) error
6777
GetComposes(ctx context.Context, orgId string, since time.Duration, limit, offset int, ignoreImageTypes []string) ([]ComposeWithBlueprintVersion, int, error)
@@ -77,9 +87,9 @@ type DB interface {
7787
GetClonesForCompose(ctx context.Context, composeId uuid.UUID, orgId string, limit, offset int) ([]CloneEntry, int, error)
7888
GetClone(ctx context.Context, id uuid.UUID, orgId string) (*CloneEntry, error)
7989

80-
InsertBlueprint(ctx context.Context, id uuid.UUID, versionId uuid.UUID, orgID, accountNumber, name, description string, body json.RawMessage, metadata json.RawMessage) error
90+
InsertBlueprint(ctx context.Context, id uuid.UUID, versionId uuid.UUID, orgID, accountNumber, name, description string, body json.RawMessage, metadata json.RawMessage, serviceSnapshots json.RawMessage) error
8191
GetBlueprint(ctx context.Context, id uuid.UUID, orgID string, version *int) (*BlueprintEntry, error)
82-
UpdateBlueprint(ctx context.Context, id uuid.UUID, blueprintId uuid.UUID, orgId string, name string, description string, body json.RawMessage) error
92+
UpdateBlueprint(ctx context.Context, id uuid.UUID, blueprintId uuid.UUID, orgId string, name string, description string, body json.RawMessage, serviceSnapshots json.RawMessage) error
8393
GetBlueprints(ctx context.Context, orgID string, limit, offset int) ([]BlueprintWithNoBody, int, error)
8494
FindBlueprints(ctx context.Context, orgID, search string, limit, offset int) ([]BlueprintWithNoBody, int, error)
8595
FindBlueprintByName(ctx context.Context, orgID, nameQuery string) (*BlueprintWithNoBody, error)

0 commit comments

Comments
 (0)