merged with component dependencies update

Author: Hubtrick-Git

.github/workflows/devguard-scanner.yaml

@@ -62,8 +62,8 @@ jobs:
     with:
       asset-name: l3montree-cybersecurity/projects/devguard/assets/devguard
       api-url: https://api.main.devguard.org
-      fail-on-risk: never
-      fail-on-cvss: never
+      fail-on-risk: high
+      fail-on-cvss: high
       web-ui: https://main.devguard.org
       continue-on-open-code-risk: true
     secrets:
@@ -85,8 +85,8 @@ jobs:
       asset-name: l3montree-cybersecurity/projects/devguard/assets/devguard
       api-url: https://api.main.devguard.org
       web-ui: https://main.devguard.org
-      fail-on-cvss: never
-      fail-on-risk: never
+      fail-on-cvss: high
+      fail-on-risk: high
       nix-cache-substituter: https://nix.garage.l3montree.cloud
       nix-cache-public-key: nix.garage.l3montree.cloud:MGlzfPQKA91/zxw91CN+GP7NpjAAwmKvWXlDYgeeI8k=
       nix-cache-s3-bucket: nix
@@ -111,8 +111,8 @@ jobs:
       asset-name: l3montree-cybersecurity/projects/devguard/assets/devguard
       api-url: https://api.main.devguard.org
       web-ui: https://main.devguard.org
-      fail-on-cvss: never
-      fail-on-risk: never
+      fail-on-cvss: high
+      fail-on-risk: high
       nix-cache-substituter: https://nix.garage.l3montree.cloud
       nix-cache-public-key: nix.garage.l3montree.cloud:MGlzfPQKA91/zxw91CN+GP7NpjAAwmKvWXlDYgeeI8k=
       nix-cache-s3-bucket: nix
@@ -136,8 +136,8 @@ jobs:
       asset-name: l3montree-cybersecurity/projects/devguard/assets/devguard-postgresql
       api-url: https://api.main.devguard.org
       web-ui: https://main.devguard.org
-      fail-on-cvss: never
-      fail-on-risk: never
+      fail-on-cvss: high
+      fail-on-risk: high
       nix-cache-substituter: https://nix.garage.l3montree.cloud
       nix-cache-public-key: nix.garage.l3montree.cloud:MGlzfPQKA91/zxw91CN+GP7NpjAAwmKvWXlDYgeeI8k=
       nix-cache-s3-bucket: nix

.github/workflows/vulndb.yaml

@@ -53,7 +53,7 @@ jobs:
       - name: Export the vulnerability database archive
         run: |
           # writes the database snapshot files and bundles them into a single tar.zst archive
-          go run ./cmd/devguard-cli/main.go vulndb export
+          go run ./cmd/devguard-cli/main.go vulndb export --diff-to-previous
 
       - name: Install Cosign
         uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0

cmd/devguard-cli/commands/vulndb_import.go

@@ -14,11 +14,11 @@ import (
 )
 
 func newImportCommand() *cobra.Command {
-	var full bool
 	var batchSize int
 	var bulk bool
 	var limitedToTables []string
 	var debug bool
+	var localArchive bool
 
 	importCmd := &cobra.Command{
 		Use:   "import",
@@ -28,11 +28,11 @@ func newImportCommand() *cobra.Command {
 			shared.LoadConfig() // nolint
 			migrateDB()
 			opts := shared.ImportOptions{
-				Full:            full,
 				BatchSize:       batchSize,
 				Bulk:            bulk,
 				LimitedToTables: limitedToTables,
 				Debug:           debug,
+				LocalArchive:    localArchive,
 			}
 			app := fx.New(
 				fx.NopLogger,
@@ -59,16 +59,19 @@ func newImportCommand() *cobra.Command {
 		},
 	}
 
-	importCmd.Flags().BoolVar(&full, "full", false, "Force a full import, ignoring the last-import watermark")
 	importCmd.Flags().IntVar(&batchSize, "batchSize", 5000, "Number of OSV entries per batch (default 5000)")
 	importCmd.Flags().BoolVar(&bulk, "bulk", false, "Load all gob data into RAM before writing (faster but uses ~2-3 GB memory)")
 	importCmd.Flags().StringSliceVar(&limitedToTables, "limitedToTables", []string{}, "Comma-separated list of tables to limit the import to (e.g. --limitedToTables=cves,exploits,malicious_packages)")
 	importCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug logging")
+	importCmd.Flags().BoolVar(&localArchive, "local-archive", false, "Read from vulndb.tar.zst in the current directory instead of pulling from OCI")
 
 	return importCmd
 }
 
 func newExportCommand() *cobra.Command {
+	var diffToPrevious bool
+	var localArchive bool
+
 	exportCmd := &cobra.Command{
 		Use:   "export",
 		Short: "Export the vulnerability database to an OCI artifact",
@@ -84,6 +87,9 @@ func newExportCommand() *cobra.Command {
 				services.ServiceModule,
 				vulndb.Module,
 				fx.Invoke(func(svc shared.VulnDBService) error {
+					if diffToPrevious {
+						return svc.ExportRCWithDiff(context.Background(), localArchive)
+					}
 					return svc.ExportRC(context.Background())
 				}),
 			)
@@ -101,5 +107,10 @@ func newExportCommand() *cobra.Command {
 		},
 	}
 
+	exportCmd.Flags().BoolVar(&diffToPrevious, "diff-to-previous", false,
+		"Compute a QuickDiff against the previous export so importers on the last version can skip staging entirely")
+	exportCmd.Flags().BoolVar(&localArchive, "local-archive", false,
+		"Use vulndb.tar.zst in the current directory for the baseline import instead of pulling from OCI")
+
 	return exportCmd
 }

cmd/devguard/main.go

@@ -294,7 +294,10 @@ func initTracer(sampleRate float64) {
 	}
 
 	opts := []sdktrace.TracerProviderOption{
-		sdktrace.WithSampler(sdktrace.TraceIDRatioBased(sampleRate)),
+		// ParentBased honors the sampled flag from an incoming traceparent header, so a specific
+		// request can be force-traced regardless of the ratio by setting the flag to 01:
+		//   curl -H "traceparent: 00-$(openssl rand -hex 16)-$(openssl rand -hex 8)-01" <url>
+		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(sampleRate))),
 		sdktrace.WithResource(res),
 	}
 	for _, p := range processors {

controllers/artifact_controller.go

@@ -109,6 +109,8 @@ func (c *ArtifactController) Create(ctx shared.Context) error {
 
 	var body requestBody
 
+	userAgent := ctx.Request().UserAgent()
+
 	if err := ctx.Bind(&body); err != nil {
 		return err
 	}
@@ -150,7 +152,7 @@ func (c *ArtifactController) Create(ctx shared.Context) error {
 	}
 	currentUserID := shared.GetSession(ctx).GetUserID()
 
-	_, _, newState, err := c.ScanNormalizedSBOM(ctx.Request().Context(), tx, org, project, asset, assetVersion, artifact, bom, currentUserID)
+	_, _, newState, err := c.ScanNormalizedSBOM(ctx.Request().Context(), tx, org, project, asset, assetVersion, artifact, bom, currentUserID, &userAgent)
 
 	if err != nil {
 		tx.Rollback()
@@ -190,7 +192,7 @@ func (c *ArtifactController) Create(ctx shared.Context) error {
 					ArtifactName: artifact.ArtifactName,
 				},
 				SBOM: exportedBOM,
-			}); err != nil {
+			}, &userAgent); err != nil {
 				slog.Error("could not handle SBOM updated event", "err", err)
 			} else {
 				slog.Info("handled SBOM updated event", "assetVersion", assetVersion.Name, "assetID", assetVersion.AssetID)
@@ -199,7 +201,7 @@ func (c *ArtifactController) Create(ctx shared.Context) error {
 	}
 
 	c.FireAndForget(func() {
-		err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, newState)
+		err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, newState, &userAgent)
 		if err != nil {
 			slog.Error("could not create issues for vulnerabilities", "err", err)
 		}
@@ -238,6 +240,8 @@ func (c *ArtifactController) DeleteArtifact(ctx shared.Context) error {
 	org := shared.GetOrg(ctx)
 	project := shared.GetProject(ctx)
 
+	userAgent := ctx.Request().UserAgent()
+
 	// we need to sync the vulnerabilities after deleting the artifact
 	// maybe we need to close some: https://github.com/l3montree-dev/devguard/issues/1496
 	// fetch all vulnerabilities which ONLY belong to this artifact
@@ -258,7 +262,7 @@ func (c *ArtifactController) DeleteArtifact(ctx shared.Context) error {
 	linkedCtx := trace.ContextWithSpan(context.Background(), trace.SpanFromContext(reqCtx))
 	if len(syncVulns) > 0 {
 		c.FireAndForget(func() {
-			err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, syncVulns)
+			err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, syncVulns, &userAgent)
 			if err != nil {
 				slog.Error("could not sync issues for vulnerabilities after artifact deletion", "err", err)
 			}
@@ -299,6 +303,7 @@ func (c *ArtifactController) UpdateArtifact(ctx shared.Context) error {
 	}
 
 	reqCtx := ctx.Request().Context()
+	userAgent := ctx.Request().UserAgent()
 
 	artifact, err := c.artifactService.ReadArtifact(reqCtx, nil, artifactName, assetVersion.Name, asset.ID)
 	if err != nil {
@@ -360,7 +365,7 @@ func (c *ArtifactController) UpdateArtifact(ctx shared.Context) error {
 		return echo.NewHTTPError(500, "could not update sbom").WithInternal(err)
 	}
 
-	_, _, vulns, err = c.ScanNormalizedSBOM(reqCtx, tx, org, project, asset, assetVersion, artifact, sbom, shared.GetSession(ctx).GetUserID())
+	_, _, vulns, err = c.ScanNormalizedSBOM(reqCtx, tx, org, project, asset, assetVersion, artifact, sbom, shared.GetSession(ctx).GetUserID(), &userAgent)
 	if err != nil {
 		slog.Error("could not scan sbom after updating it", "err", err)
 		return echo.NewHTTPError(500, "could not scan sbom after updating it").WithInternal(err)
@@ -395,7 +400,7 @@ func (c *ArtifactController) UpdateArtifact(ctx shared.Context) error {
 					ArtifactName: artifactName,
 				},
 				SBOM: exportedBOM,
-			}); err != nil {
+			}, &userAgent); err != nil {
 				slog.Error("could not handle SBOM updated event", "err", err)
 			} else {
 				slog.Info("handled SBOM updated event", "assetVersion", assetVersion.Name, "assetID", assetVersion.AssetID)
@@ -404,7 +409,7 @@ func (c *ArtifactController) UpdateArtifact(ctx shared.Context) error {
 	}
 
 	c.FireAndForget(func() {
-		err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns)
+		err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns, &userAgent)
 		if err != nil {
 			slog.Error("could not create issues for vulnerabilities", "err", err)
 		}

controllers/asset_controller.go

@@ -329,6 +329,7 @@ func (a *AssetController) Update(ctx shared.Context) error {
 
 	org := shared.GetOrg(ctx)
 	project := shared.GetProject(ctx)
+	userAgent := ctx.Request().UserAgent()
 	if enableTicketRangeUpdated || justification != "" {
 		//check if we have already created the labels in gitlab, if not create them
 		// do NOT update the asset in the database yet, we do this after the ticket sync
@@ -353,7 +354,7 @@ func (a *AssetController) Update(ctx shared.Context) error {
 				return
 			}
 
-			if err := a.dependencyVulnService.SyncAllIssues(linkedCtx, org, project, asset, defaultAssetVersion); err != nil {
+			if err := a.dependencyVulnService.SyncAllIssues(linkedCtx, org, project, asset, defaultAssetVersion, &userAgent); err != nil {
 				slog.Warn("could not sync tickets", "err", err)
 			}
 		})

controllers/dependency_vuln_controller.go

@@ -233,11 +233,11 @@ func (controller DependencyVulnController) Mitigate(ctx shared.Context) error {
 	}
 
 	thirdPartyIntegrations := shared.GetThirdPartyIntegration(ctx)
-
+	userAgent := ctx.Request().UserAgent()
 	if err = thirdPartyIntegrations.HandleEvent(ctx.Request().Context(), shared.ManualMitigateEvent{
 		Ctx:           ctx,
 		Justification: j.Comment,
-	}); err != nil {
+	}, &userAgent); err != nil {
 		return echo.NewHTTPError(500, "could not mitigate dependencyVuln").WithInternal(err)
 	}
 
@@ -309,6 +309,7 @@ func (controller DependencyVulnController) SyncDependencyVulns(ctx shared.Contex
 	assetVersion := shared.GetAssetVersion(ctx)
 	org := shared.GetOrg(ctx)
 	project := shared.GetProject(ctx)
+	userAgent := ctx.Request().UserAgent()
 
 	type vulnReq struct {
 		VulnID uuid.UUID         `json:"vulnId"`
@@ -355,7 +356,7 @@ func (controller DependencyVulnController) SyncDependencyVulns(ctx shared.Contex
 
 	linkedCtx := trace.ContextWithSpan(context.Background(), trace.SpanFromContext(ctx.Request().Context()))
 	controller.FireAndForget(func() {
-		err := controller.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns)
+		err := controller.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns, &userAgent)
 		if err != nil {
 			slog.Error("could not create issues for vulnerabilities", "err", err)
 		}
@@ -405,7 +406,8 @@ func (controller DependencyVulnController) CreateEvent(ctx shared.Context) error
 	justification := status.Justification
 	mechanicalJustification := status.MechanicalJustification
 
-	ev, err := controller.dependencyVulnService.CreateVulnEventAndApply(ctx.Request().Context(), nil, asset.ID, userID, &dependencyVuln, dtos.VulnEventType(statusType), justification, mechanicalJustification, assetVersion.Name)
+	userAgent := ctx.Request().UserAgent()
+	ev, err := controller.dependencyVulnService.CreateVulnEventAndApply(ctx.Request().Context(), nil, asset.ID, userID, &dependencyVuln, dtos.VulnEventType(statusType), justification, mechanicalJustification, assetVersion.Name, &userAgent)
 	if err != nil {
 		return err
 	}
@@ -424,7 +426,7 @@ func (controller DependencyVulnController) CreateEvent(ctx shared.Context) error
 	err = thirdPartyIntegration.HandleEvent(ctx.Request().Context(), shared.VulnEvent{
 		Ctx:   ctx,
 		Event: ev,
-	})
+	}, &userAgent)
 	// we do not want the transaction to be rolled back if the third party integration fails
 	if err != nil {
 		// just log the error
@@ -469,7 +471,8 @@ func (controller DependencyVulnController) BatchCreateEvent(ctx shared.Context)
 			continue
 		}
 
-		ev, err := controller.dependencyVulnService.CreateVulnEventAndApply(ctx.Request().Context(), nil, asset.ID, userID, &dependencyVuln, eventType, status.Justification, status.MechanicalJustification, assetVersion.Name)
+		userAgent := ctx.Request().UserAgent()
+		ev, err := controller.dependencyVulnService.CreateVulnEventAndApply(ctx.Request().Context(), nil, asset.ID, userID, &dependencyVuln, eventType, status.Justification, status.MechanicalJustification, assetVersion.Name, &userAgent)
 		if err != nil {
 			slog.Error("could not create event for dependencyVuln", "err", err, "vulnID", vulnID)
 			continue
@@ -486,7 +489,7 @@ func (controller DependencyVulnController) BatchCreateEvent(ctx shared.Context)
 		if err := thirdPartyIntegration.HandleEvent(ctx.Request().Context(), shared.VulnEvent{
 			Ctx:   ctx,
 			Event: ev,
-		}); err != nil {
+		}, &userAgent); err != nil {
 			slog.Error("could not handle event", "err", err)
 		}
 

controllers/external_reference_controller.go

@@ -187,6 +187,7 @@ func (c *ExternalReferenceController) Sync(ctx shared.Context) error {
 	org := shared.GetOrg(ctx)
 	project := shared.GetProject(ctx)
 	userID := shared.GetSession(ctx).GetUserID()
+	userAgent := ctx.Request().UserAgent()
 
 	artifacts, err := c.artifactRepository.GetByAssetIDAndAssetVersionName(ctx.Request().Context(), nil, asset.ID, assetVersion.Name)
 	if err != nil {
@@ -198,7 +199,7 @@ func (c *ExternalReferenceController) Sync(ctx shared.Context) error {
 		tx := c.artifactRepository.Begin(ctx.Request().Context())
 		defer tx.Rollback()
 
-		_, _, vulns, err := c.RunArtifactSecurityLifecycle(ctx.Request().Context(), tx, org, project, asset, assetVersion, artifact, userID)
+		_, _, vulns, err := c.RunArtifactSecurityLifecycle(ctx.Request().Context(), tx, org, project, asset, assetVersion, artifact, userID, &userAgent)
 		if err != nil {
 			tx.Rollback()
 			slog.Error("could not scan sbom after syncing external sources", "err", err, "artifact", artifact.ArtifactName)
@@ -213,7 +214,7 @@ func (c *ExternalReferenceController) Sync(ctx shared.Context) error {
 
 		linkedCtx := trace.ContextWithSpan(context.Background(), trace.SpanFromContext(ctx.Request().Context()))
 		c.FireAndForget(func() {
-			if err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns); err != nil {
+			if err := c.dependencyVulnService.SyncIssues(linkedCtx, org, project, asset, assetVersion, vulns, &userAgent); err != nil {
 				slog.Error("could not create issues for vulnerabilities", "err", err)
 			}
 		})

controllers/first_party_vuln_controller.go

@@ -105,6 +105,8 @@ func (c FirstPartyVulnController) Mitigate(ctx shared.Context) error {
 		return echo.NewHTTPError(400, "invalid firstPartyVulnID")
 	}
 
+	userAgent := ctx.Request().UserAgent()
+
 	thirdPartyIntegrations := shared.GetThirdPartyIntegration(ctx)
 
 	var j struct {
@@ -119,7 +121,7 @@ func (c FirstPartyVulnController) Mitigate(ctx shared.Context) error {
 	if err = thirdPartyIntegrations.HandleEvent(ctx.Request().Context(), shared.ManualMitigateEvent{
 		Justification: j.Justification,
 		Ctx:           ctx,
-	}); err != nil {
+	}, &userAgent); err != nil {
 		return echo.NewHTTPError(500, "could not mitigate firstPartyVuln").WithInternal(err)
 	}
 
@@ -190,15 +192,16 @@ func (c FirstPartyVulnController) CreateEvent(ctx shared.Context) error {
 
 	mechanicalJustification := status.MechanicalJustification
 
-	ev, err := c.firstPartyVulnService.UpdateFirstPartyVulnState(ctx.Request().Context(), nil, userID, &firstPartyVuln, statusType, justification, mechanicalJustification)
+	userAgent := ctx.Request().UserAgent()
+	ev, err := c.firstPartyVulnService.UpdateFirstPartyVulnState(ctx.Request().Context(), nil, userID, &firstPartyVuln, statusType, justification, mechanicalJustification, &userAgent)
 	if err != nil {
 		return err
 	}
 
 	err = thirdPartyIntegration.HandleEvent(ctx.Request().Context(), shared.VulnEvent{
 		Ctx:   ctx,
 		Event: ev,
-	})
+	}, &userAgent)
 	// we do not want the transaction to be rolled back if the third party integration fails
 	if err != nil {
 		// just log the error