Skip to content

Commit 459a95a

Browse files
Hubtrick-GitHubtrick-Git
committed
migrate component dependencies to have a new composite primary key and remove obsolete indexes and columns
1 parent 571ee88 commit 459a95a

10 files changed

Lines changed: 57 additions & 22 deletions

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,5 @@ key.pem
3232
cache
3333
CLAUDE.md
3434
.claudeignore
35-
result
35+
result
36+
instance_admin_pub_key_new.pem

‎database/migrations/20260518100544_remove_obsolete_indexes_and_columns_from_component_dependencies.down.sql‎

Whitespace-only changes.

‎database/migrations/20260518100544_remove_obsolete_indexes_and_columns_from_component_dependencies.up.sql‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
ALTER TABLE public.component_dependencies
2+
DROP COLUMN IF EXISTS semver_start,
3+
DROP COLUMN IF EXISTS semver_end;
4+
5+
DROP INDEX IF EXISTS idx_component_dependencies_component_purl;
6+
DROP INDEX IF EXISTS component_purl_idx;
7+
DROP INDEX IF EXISTS asset_version_name_idx;
8+
DROP INDEX IF EXISTS idx_component_dependencies_dependency_purl;
9+
DROP INDEX IF EXISTS idx_component_dependencies_null_roots;
10+
DROP INDEX IF EXISTS asset_id_idx;
11+
DROP INDEX IF EXISTS idx_component_dependencies_component_id;
12+
DROP INDEX IF EXISTS idx_component_dependencies_dependency_id;
13+
DROP INDEX IF EXISTS idx_component_dependencies_recursive_lookup;
14+
DROP INDEX IF EXISTS dependency_purl_idx;
15+
16+
-- remove the current id column and replace it with a composite key on all columns to make enforce deduplication on a data level and reduce complexity (also on indexes)
17+
18+
-- currently component_id can be NULL if its a root node; but primary keys cannot contain NULL values, so we replace it with a ROOT constant
19+
INSERT INTO public.components VALUES('ROOT','',NULL,NULL,NULL); -- add a ROOT component to the components table to be referenced
20+
UPDATE public.component_dependencies SET component_id = 'ROOT' WHERE component_id IS NULL; -- use an explicit value for ROOT component dependencies instead of NULL
21+
22+
ALTER TABLE public.component_dependencies DROP CONSTRAINT component_dependencies_pkey, DROP COLUMN id; -- drop primary key constraint and the primary key column
23+
24+
-- remove all duplicate entries from the table so that the new primary key does not fail on creation
25+
DELETE FROM public.component_dependencies a
26+
USING public.component_dependencies b
27+
WHERE a.ctid < b.ctid -- use the internal column id to choose only 1 candidate per duplicate row
28+
AND a.asset_id = b.asset_id
29+
AND a.asset_version_name = b.asset_version_name
30+
AND a.dependency_id = b.dependency_id
31+
AND a.component_id = b.component_id;
32+
33+
ALTER TABLE public.component_dependencies ADD PRIMARY KEY (asset_id, asset_version_name, dependency_id, component_id); -- add the new primary key consisting of all 4 attributes

‎database/models/component_model.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ type ComponentDependency struct {
7070
// this means, that the dependency graph between people using the same library might differ, since they use it differently
7171
// we use edges, which provide the information, that a component is used by another component in one asset
7272
Component Component `json:"component" gorm:"foreignKey:ComponentID;references:ID;constraint:OnDelete:CASCADE;"`
73-
ComponentID *string `json:"componentPurl" gorm:"column:component_id;index:component_purl_idx"` // will be nil, for direct dependencies
73+
ComponentID *string `json:"componentPurl" gorm:"column:component_id;index:component_purl_idx"` // will be ROOT, for direct dependencies
7474
Dependency Component `json:"dependency" gorm:"foreignKey:DependencyID;references:ID;constraint:OnDelete:CASCADE;"`
7575
DependencyID string `json:"dependencyPurl" gorm:"column:dependency_id;index:dependency_purl_idx"`
7676

‎database/repositories/component_repository.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func (c *componentRepository) LoadComponentsWithProject(ctx context.Context, tx
147147
componentDependencies[i].Dependency.License = &license
148148
componentDependencies[i].Dependency.IsLicenseOverwritten = true
149149
}
150-
if component.ComponentID != nil {
150+
if component.ComponentID != nil && *component.ComponentID != "ROOT" {
151151
if license, ok := isPurlOverwrittenMap[*component.ComponentID]; ok {
152152
componentDependencies[i].Component.License = &license
153153
componentDependencies[i].Component.IsLicenseOverwritten = true
@@ -232,18 +232,18 @@ func (c *componentRepository) HandleStateDiff(ctx context.Context, tx *gorm.DB,
232232
for _, edge := range diff.AddedEdges {
233233
c1 := wholeAssetGraph.Node(edge[0])
234234
c2 := wholeAssetGraph.Node(edge[1])
235-
var componentID *string
235+
var componentID string
236236
if c1.Type == normalize.GraphNodeTypeRoot {
237-
// set to nil for root nodes
238-
componentID = nil
237+
// set to ROOT for root nodes
238+
componentID = "ROOT"
239239
} else {
240-
componentID = utils.Ptr(c1.Component.PackageURL)
240+
componentID = c1.Component.PackageURL
241241
}
242242

243243
componentDependency := models.ComponentDependency{
244244
AssetID: assetVersion.AssetID,
245245
AssetVersionName: assetVersion.Name,
246-
ComponentID: componentID,
246+
ComponentID: utils.Ptr(componentID),
247247
DependencyID: c2.Component.PackageURL,
248248
}
249249

‎database/repositories/gorm_repository.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,10 +234,10 @@ WHERE NOT EXISTS (SELECT artifact_dependency_vulns.dependency_vuln_id FROM artif
234234
DELETE FROM license_risks lr
235235
WHERE NOT EXISTS (SELECT artifact_license_risks.license_risk_id FROM artifact_license_risks WHERE artifact_license_risks.license_risk_id = lr.id);
236236
237-
-- Clean up artifact root nodes (component_id IS NULL, dependency_id LIKE 'artifact:%')
237+
-- Clean up artifact root nodes (component_id = 'ROOT', dependency_id LIKE 'artifact:%')
238238
-- where the artifact no longer exists
239239
DELETE FROM component_dependencies cd
240-
WHERE cd.component_id IS NULL
240+
WHERE cd.component_id = 'ROOT'
241241
AND cd.dependency_id LIKE 'artifact:%'
242242
AND NOT EXISTS (
243243
SELECT 1 FROM artifacts a

‎integrations/commonint/integration_helper_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ func TestRenderPathToComponent(t *testing.T) {
398398
t.Run("Everything works as expeted with a non empty component list", func(t *testing.T) {
399399
// Create a chain of actual components (all with pkg: prefix) to have a path with edges
400400
components := []models.ComponentDependency{
401-
{ComponentID: nil, DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}}, // root --> artifact
401+
{ComponentID: utils.Ptr("ROOT"), DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}}, // root --> artifact
402402
{ComponentID: utils.Ptr("artifact:test-artifact"), DependencyID: "sbom:test@test-artifact", Dependency: models.Component{ID: "sbom:test@test-artifact"}}, // artifact -> sbom
403403
{ComponentID: utils.Ptr("sbom:test@test-artifact"), DependencyID: "pkg:npm/root-dep@1.0.0", Dependency: models.Component{ID: "pkg:npm/root-dep@1.0.0"}}, // sbom -> root-dep (component)
404404
{ComponentID: utils.Ptr("pkg:npm/root-dep@1.0.0"), DependencyID: "pkg:npm/test-package@1.0.0", Dependency: models.Component{ID: "pkg:npm/test-package@1.0.0"}}, // root-dep -> test-package (component)
@@ -423,7 +423,7 @@ func TestRenderPathToComponent(t *testing.T) {
423423
t.Run("should escape @ symbols", func(t *testing.T) {
424424
// Create a chain of actual components to verify @ escaping in mermaid output
425425
components := []models.ComponentDependency{
426-
{ComponentID: nil, DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}}, // root --> artifact
426+
{ComponentID: utils.Ptr("ROOT"), DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}}, // root --> artifact
427427
{ComponentID: utils.Ptr("artifact:test-artifact"), DependencyID: "sbom:test@test-artifact", Dependency: models.Component{ID: "sbom:test@test-artifact"}}, // artifact -> sbom
428428
{ComponentID: utils.Ptr("sbom:test@test-artifact"), DependencyID: "pkg:npm/root-dep@1.0.0", Dependency: models.Component{ID: "pkg:npm/root-dep@1.0.0"}}, // sbom -> root-dep (component)
429429
{ComponentID: utils.Ptr("pkg:npm/root-dep@1.0.0"), DependencyID: "pkg:npm/test-package@1.0.0", Dependency: models.Component{ID: "pkg:npm/test-package@1.0.0"}}, // root-dep -> test-package (component)
@@ -452,7 +452,7 @@ func TestRenderPathToComponent(t *testing.T) {
452452
// The graph requires an artifact and sbom info-source node above the component
453453
// so that FindAllComponentOnlyPathsToPURL can terminate the BFS correctly.
454454
components := []models.ComponentDependency{
455-
{ComponentID: nil, DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}},
455+
{ComponentID: utils.Ptr("ROOT"), DependencyID: "artifact:test-artifact", Dependency: models.Component{ID: "artifact:test-artifact"}},
456456
{ComponentID: utils.Ptr("artifact:test-artifact"), DependencyID: "sbom:test@test-artifact", Dependency: models.Component{ID: "sbom:test@test-artifact"}},
457457
{ComponentID: utils.Ptr("sbom:test@test-artifact"), DependencyID: "pkg:npm/single@1.0.0", Dependency: models.Component{ID: "pkg:npm/single@1.0.0"}},
458458
}
@@ -711,7 +711,7 @@ func TestTicketContentBitwiseReproducibility(t *testing.T) {
711711
// Previously, map iteration randomness caused the two path edges to appear
712712
// in non-deterministic order in the Mermaid output.
713713
components := []models.ComponentDependency{
714-
{ComponentID: nil, DependencyID: "artifact:art", Dependency: models.Component{ID: "artifact:art"}},
714+
{ComponentID: utils.Ptr("ROOT"), DependencyID: "artifact:art", Dependency: models.Component{ID: "artifact:art"}},
715715
{ComponentID: utils.Ptr("artifact:art"), DependencyID: "sbom:s@art", Dependency: models.Component{ID: "sbom:s@art"}},
716716
{ComponentID: utils.Ptr("sbom:s@art"), DependencyID: "pkg:npm/route-b@1.0", Dependency: models.Component{ID: "pkg:npm/route-b@1.0"}},
717717
{ComponentID: utils.Ptr("sbom:s@art"), DependencyID: "pkg:npm/route-a@1.0", Dependency: models.Component{ID: "pkg:npm/route-a@1.0"}},

‎tests/component_service_integration_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func TestGetAndSaveLicenseInformation(t *testing.T) {
9797
err = f.DB.Create(&models.ComponentDependency{
9898
AssetVersionName: assetVersion.Name,
9999
AssetID: assetVersion.AssetID,
100-
ComponentID: nil,
100+
ComponentID: utils.Ptr("ROOT"),
101101
DependencyID: artifactRoot,
102102
}).Error
103103
assert.NoError(t, err)
@@ -219,7 +219,7 @@ func TestGetAndSaveLicenseInformation(t *testing.T) {
219219
err = f.DB.Create(&models.ComponentDependency{
220220
AssetVersionName: assetVersion.Name,
221221
AssetID: assetVersion.AssetID,
222-
ComponentID: nil,
222+
ComponentID: utils.Ptr("ROOT"),
223223
DependencyID: artifactRoot,
224224
}).Error
225225
assert.NoError(t, err)

‎tests/daemon_pipeline_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func createSBOMStructure(f *TestFixture, asset models.Asset, assetVersion models
5151
artifactRootDep := models.ComponentDependency{
5252
AssetID: asset.ID,
5353
AssetVersionName: assetVersion.Name,
54-
ComponentID: nil,
54+
ComponentID: utils.Ptr("ROOT"),
5555
DependencyID: artifactRoot,
5656
}
5757
if err := f.DB.Create(&artifactRootDep).Error; err != nil {
@@ -179,7 +179,7 @@ func TestDaemonPipelineEndToEnd(t *testing.T) {
179179
artifactRootDep := models.ComponentDependency{
180180
AssetID: asset.ID,
181181
AssetVersionName: assetVersion.Name,
182-
ComponentID: nil,
182+
ComponentID: utils.Ptr("ROOT"),
183183
DependencyID: artifactRoot,
184184
}
185185
err = f.DB.Create(&artifactRootDep).Error
@@ -580,7 +580,7 @@ func TestDaemonPipelineScanAssetDetectVulns(t *testing.T) {
580580
artifactRootDep := models.ComponentDependency{
581581
AssetID: asset.ID,
582582
AssetVersionName: assetVersion.Name,
583-
ComponentID: nil,
583+
ComponentID: utils.Ptr("ROOT"),
584584
DependencyID: artifactRoot,
585585
}
586586
err = f.DB.Create(&artifactRootDep).Error
@@ -772,7 +772,7 @@ func TestDaemonPipelineRiskCalculation(t *testing.T) {
772772
artifactRootDep := models.ComponentDependency{
773773
AssetID: asset.ID,
774774
AssetVersionName: assetVersion.Name,
775-
ComponentID: nil,
775+
ComponentID: utils.Ptr("ROOT"),
776776
DependencyID: artifactRoot,
777777
}
778778
err = f.DB.Create(&artifactRootDep).Error

‎tests/release_integration_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/l3montree-dev/devguard/controllers"
2727
"github.com/l3montree-dev/devguard/database/models"
2828
"github.com/l3montree-dev/devguard/shared"
29+
"github.com/l3montree-dev/devguard/utils"
2930
"github.com/labstack/echo/v4"
3031
"github.com/stretchr/testify/assert"
3132
)
@@ -79,8 +80,8 @@ func TestReleaseSBOMMergeIntegration(t *testing.T) {
7980
ID: artifactRoot2,
8081
})
8182

82-
rootDep1 := models.ComponentDependency{DependencyID: artifactRoot1, AssetVersionName: assetVersion.Name, AssetID: asset.ID, ComponentID: nil}
83-
rootDep2 := models.ComponentDependency{DependencyID: artifactRoot2, AssetVersionName: assetVersion.Name, AssetID: asset.ID, ComponentID: nil}
83+
rootDep1 := models.ComponentDependency{DependencyID: artifactRoot1, AssetVersionName: assetVersion.Name, AssetID: asset.ID, ComponentID: utils.Ptr("ROOT")}
84+
rootDep2 := models.ComponentDependency{DependencyID: artifactRoot2, AssetVersionName: assetVersion.Name, AssetID: asset.ID, ComponentID: utils.Ptr("ROOT")}
8485
if err := f.DB.Create(&rootDep1).Error; err != nil {
8586
t.Fatal(err)
8687
}

0 commit comments

Comments
 (0)