Merge remote-tracking branch 'origin/main' into 1720-add-config-option-to-disable-organisation-creation-for-an-devguard-instance

Author: refoo0

.github/workflows/devguard-scanner.yaml

@@ -46,7 +46,7 @@ jobs:
           substituters = https://cache.nixos.org https://nix.garage.l3montree.cloud
           trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.garage.l3montree.cloud:MGlzfPQKA91/zxw91CN+GP7NpjAAwmKvWXlDYgeeI8k=
     - name: Run unittests
-      run: nix develop . --command bash -c "go test \$(go list ./... | grep -v '/mocks') -coverprofile=coverage.out && go tool cover -func=coverage.out"
+      run: nix develop . --command bash -c "go test -timeout 20m \$(go list ./... | grep -v '/mocks') -coverprofile=coverage.out && go tool cover -func=coverage.out"
     - name: Archive code coverage results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - https://github.com/actions/upload-artifact/releases/tag/v4.6.2
       with:
@@ -121,7 +121,7 @@ jobs:
       devguard-token: ${{ secrets.DEVGUARD_TOKEN }}
 
   postgresql-pipeline:
-    if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main'
+    if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/')
     uses: l3montree-dev/devguard-action/.github/workflows/full-nix.yml@nix
     permissions:
       contents: read

.github/workflows/vulndb.yaml

@@ -22,14 +22,14 @@ jobs:
       FRONTEND_URL: "doesntmatter"
     services:
       postgres:
-        image: ghcr.io/l3montree-dev/devguard-postgresql:v0.5.3@sha256:a06c9e7c8ee334790cc66d52e89ff5ef05352ab264841d3d9f3659c046732251
+        image: ghcr.io/l3montree-dev/devguard/postgresql:v1.3.1
         env:
           POSTGRES_DB: ${{env.POSTGRES_DB}}
           POSTGRES_USER: ${{env.POSTGRES_USER}}
           POSTGRES_PASSWORD: ${{env.POSTGRES_PASSWORD}}
         ports:
           - 5432:5432
-        options: '--health-cmd="pg_isready -U devguard"  --health-interval=10s  --health-timeout=5s  --health-retries=5 '
+        options: '--health-cmd="pg_isready -U devguard" --health-interval=10s --health-timeout=5s --health-retries=5 --tmpfs /docker-entrypoint-initdb.d --tmpfs /run/postgresql'
     steps:
       - name: Install postgresql client
         run: |

accesscontrol/members.go

@@ -14,72 +14,3 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 package accesscontrol
-
-import (
-	"maps"
-
-	"github.com/l3montree-dev/devguard/dtos"
-	"github.com/l3montree-dev/devguard/shared"
-	"github.com/l3montree-dev/devguard/utils"
-	"github.com/ory/client-go"
-)
-
-// FetchMembersOfOrganization retrieves all members of an organization including their roles
-// from both the RBAC system and third-party integrations
-func FetchMembersOfOrganization(ctx shared.Context) ([]dtos.UserDTO, error) {
-	// get all members from the organization
-	organization := shared.GetOrg(ctx)
-	accessControl := shared.GetRBAC(ctx)
-
-	members, err := accessControl.GetAllMembersOfOrganization()
-
-	if err != nil {
-		return nil, err
-	}
-
-	users := make([]dtos.UserDTO, 0, len(members))
-	if len(members) > 0 {
-		// get the auth admin client from the context
-		authAdminClient := shared.GetAuthAdminClient(ctx)
-		// fetch the users from the auth service
-		m, err := authAdminClient.ListUser(client.IdentityAPIListIdentitiesRequest{}.Ids(members))
-		if err != nil {
-			return nil, err
-		}
-
-		// get the roles for the members
-		errGroup := utils.ErrGroup[map[string]shared.Role](10)
-		for _, member := range m {
-			errGroup.Go(func() (map[string]shared.Role, error) {
-				role, err := accessControl.GetDomainRole(member.Id)
-				if err != nil {
-					return map[string]shared.Role{member.Id: shared.RoleUnknown}, nil
-				}
-				return map[string]shared.Role{member.Id: role}, nil
-			})
-		}
-
-		roles, err := errGroup.WaitAndCollect()
-		if err != nil {
-			return nil, err
-		}
-
-		roleMap := utils.Reduce(roles, func(acc map[string]shared.Role, r map[string]shared.Role) map[string]shared.Role {
-			maps.Copy(acc, r)
-			return acc
-		}, make(map[string]shared.Role))
-
-		for _, member := range m {
-			users = append(users, dtos.UserDTO{
-				ID:   member.Id,
-				Name: shared.IdentityName(member.Traits),
-				Role: string(roleMap[member.Id]),
-			})
-		}
-	}
-
-	// fetch all members from third party integrations
-	thirdPartyIntegrations := shared.GetThirdPartyIntegration(ctx)
-	users = append(users, thirdPartyIntegrations.GetUsers(organization)...)
-	return users, nil
-}

cmd/devguard-cli/commands/vulndb_import.go

@@ -14,13 +14,24 @@ import (
 )
 
 func newImportCommand() *cobra.Command {
+	var full bool
+	var batchSize int
+	var bulk bool
+	var limitedToTables []string
+
 	importCmd := &cobra.Command{
 		Use:   "import",
 		Short: "Import the latest state of the vulnerability database",
 		Long:  "Pulls the pre-built vulndb artifact from the OCI registry and applies all changes to the local database",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			shared.LoadConfig() // nolint
 			migrateDB()
+			opts := shared.ImportOptions{
+				Full:            full,
+				BatchSize:       batchSize,
+				Bulk:            bulk,
+				LimitedToTables: limitedToTables,
+			}
 			app := fx.New(
 				fx.NopLogger,
 				database.Module,
@@ -29,7 +40,7 @@ func newImportCommand() *cobra.Command {
 				services.ServiceModule,
 				vulndb.Module,
 				fx.Invoke(func(svc shared.VulnDBService) error {
-					return svc.ImportRC(context.Background())
+					return svc.ImportRC(context.Background(), opts)
 				}),
 			)
 
@@ -46,6 +57,11 @@ func newImportCommand() *cobra.Command {
 		},
 	}
 
+	importCmd.Flags().BoolVar(&full, "full", false, "Force a full import, ignoring the last-import watermark")
+	importCmd.Flags().IntVar(&batchSize, "batchSize", 5000, "Number of OSV entries per batch (default 5000)")
+	importCmd.Flags().BoolVar(&bulk, "bulk", false, "Load all gob data into RAM before writing (faster but uses ~2-3 GB memory)")
+	importCmd.Flags().StringSliceVar(&limitedToTables, "limitedToTables", []string{}, "Comma-separated list of tables to limit the import to (e.g. --limitedToTables=cves,exploits,malicious_packages)")
+
 	return importCmd
 }
 

cmd/devguard-cli/hashmigrations/hash_migration.go

@@ -91,7 +91,7 @@ func RunHashMigrationsIfNeeded(pool *pgxpool.Pool, daemonRunner shared.DaemonRun
 				slog.Warn("could not clear vulndb.lastRCImport config", "err", err)
 			}
 			slog.Info("triggering full vulndb import after hash migration")
-			if err := vulndbService.ImportRC(ctx); err != nil {
+			if err := vulndbService.ImportRC(ctx, shared.ImportOptions{}); err != nil {
 				return fmt.Errorf("full vulndb import after hash migration failed: %w", err)
 			}
 		}

cmd/devguard-scanner/commands/clean.go

@@ -1,7 +1,6 @@
 package commands
 
 import (
-	"context"
 	"log/slog"
 
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -12,10 +11,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func cleanImage(ctx context.Context, regOpts cosignoptions.RegistryOptions, cleanType cosignoptions.CleanType, imageRef string) error {
-	return cosignclean.CleanCmd(ctx, regOpts, cleanType, imageRef, true)
-}
-
 // NewCleanCommand returns a command that removes attestations/signatures from an OCI image.
 func NewCleanCommand() *cobra.Command {
 	cmd := &cobra.Command{

cmd/devguard-scanner/commands/clean_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/registry"
 	"github.com/google/go-containerregistry/pkg/v1/random"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	cosignclean "github.com/sigstore/cosign/v2/cmd/cosign/cli"
 	cosignoptions "github.com/sigstore/cosign/v2/cmd/cosign/cli/options"
 	ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote"
 )
@@ -100,7 +101,7 @@ func TestCleanImageCleanTypeAllRemovesAllTags(t *testing.T) {
 		t.Fatal("signature tag should exist before clean")
 	}
 
-	if err := cleanImage(context.Background(), regOpts, cosignoptions.CleanTypeAll, imageRef); err != nil {
+	if err := cosignclean.CleanCmd(context.Background(), regOpts, cosignoptions.CleanTypeAll, imageRef, true); err != nil {
 		t.Fatalf("cleanImage: %v", err)
 	}
 
@@ -130,7 +131,7 @@ func TestCleanImageCleanTypeAttestationLeavesSignatureTag(t *testing.T) {
 	pushDummyImage(t, attTag, remoteOpts)
 	pushDummyImage(t, sigTag, remoteOpts)
 
-	if err := cleanImage(context.Background(), regOpts, cosignoptions.CleanTypeAttestation, imageRef); err != nil {
+	if err := cosignclean.CleanCmd(context.Background(), regOpts, cosignoptions.CleanTypeAttestation, imageRef, true); err != nil {
 		t.Fatalf("cleanImage: %v", err)
 	}
 
@@ -160,7 +161,7 @@ func TestCleanImageCleanTypeSignatureLeavesAttestationTag(t *testing.T) {
 	pushDummyImage(t, attTag, remoteOpts)
 	pushDummyImage(t, sigTag, remoteOpts)
 
-	if err := cleanImage(context.Background(), regOpts, cosignoptions.CleanTypeSignature, imageRef); err != nil {
+	if err := cosignclean.CleanCmd(context.Background(), regOpts, cosignoptions.CleanTypeSignature, imageRef, true); err != nil {
 		t.Fatalf("cleanImage: %v", err)
 	}
 
@@ -179,7 +180,7 @@ func TestCleanImageNoTagsDoesNotError(t *testing.T) {
 	pushBaseImage(t, imageRef, remoteOpts)
 
 	// no attestation/signature tags present — clean should still succeed
-	if err := cleanImage(context.Background(), regOpts, cosignoptions.CleanTypeAll, imageRef); err != nil {
+	if err := cosignclean.CleanCmd(context.Background(), regOpts, cosignoptions.CleanTypeAll, imageRef, true); err != nil {
 		t.Fatalf("cleanImage on image with no sig/att tags failed: %v", err)
 	}
 }

cmd/devguard-scanner/commands/intoto/intoto_record.go

@@ -20,41 +20,14 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
-	"strings"
 
 	toto "github.com/in-toto/in-toto-golang/in_toto"
 	"github.com/l3montree-dev/devguard/cmd/devguard-scanner/config"
 	"github.com/l3montree-dev/devguard/cmd/devguard-scanner/scanner"
-	"github.com/l3montree-dev/devguard/utils"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
-func parseGitIgnore(path string) ([]string, error) {
-	// read .gitignore if exists
-	content, err := os.ReadFile(path)
-	if err == nil {
-		ignorePaths := strings.Split(string(content), "\n")
-
-		// make sure to remove new lines and empty strings
-		ignorePaths = utils.Filter(
-			utils.Map(utils.Map(ignorePaths, strings.TrimSpace), func(e string) string {
-				// nextjs products a gitignore which contains /node_modules but we need to ignore /node_modules/
-				if e == "/node_modules" {
-					return e + "/"
-				}
-				return e
-			}),
-			func(e string) bool {
-				return e != "" && e != "\n" && !strings.HasPrefix(strings.TrimSpace(e), "#")
-			})
-
-		return ignorePaths, nil
-	}
-
-	return nil, err
-}
-
 func stopInTotoRecording(cmd *cobra.Command, args []string) error {
 	if config.RuntimeInTotoConfig.Disabled {
 		return nil

cmd/devguard-scanner/commands/intoto/intoto_record_test.go

@@ -1,32 +1 @@
 package intotocmd
-
-import (
-	"os"
-	"path"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseGitIgnore(t *testing.T) {
-	t.Run("parseGitIgnore with empty strings", func(t *testing.T) {
-		// create temp dir for testing
-		dir, err := os.MkdirTemp("", "test")
-		assert.NoError(t, err, "failed to create temporary directory")
-
-		defer os.RemoveAll(dir)
-
-		// Create a temporary .gitignore file for testing
-		gitignoreContent := "\n.DS_Store\n\t\t\t\n"
-
-		filepath := path.Join(dir, ".gitignore")
-
-		err = os.WriteFile(filepath, []byte(gitignoreContent), 0600)
-		assert.NoError(t, err, "failed to create temporary .gitignore file")
-
-		ignorePaths, err := parseGitIgnore(filepath)
-		assert.NoError(t, err, "expected no error when reading .gitignore")
-		assert.Equal(t, []string{".DS_Store"}, ignorePaths, "unexpected ignore paths")
-
-	})
-}

controllers/artifact_controller.go

@@ -672,7 +672,7 @@ func (c *ArtifactController) BuildVulnerabilityReportPDF(ctx shared.Context) err
 					SourceName:          escapeLatex(v.Source.Name),
 					SourceURL:           escapeLatex(v.Source.URL),
 					AffectedComponent:   escapeLatex(dv.ComponentPurl),
-					CveDescription:      escapeLatex(dv.CVE.Description),
+					CveDescription:      escapeLatex(dv.GetCVE().Description),
 					AnalysisState:       escapeLatex(string(v.Analysis.State)),
 					AnalysisResponse:    escapeLatex(response),
 					AnalysisDetail:      escapeLatex(v.Analysis.Detail),