Rework vulndb import

.github/workflows/vulndb.yaml

@@ -2,11 +2,6 @@ name: VulnDB Workflow
 
 on:
   workflow_dispatch:
-    inputs:
-      run_generate_snapshot:
-        description: "Run generate snapshot job"
-        required: false
-        default: "false"
   schedule:
     - cron: "0 */6 * * *" # every hour
 
@@ -56,39 +51,11 @@ jobs:
         uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - https://github.com/actions/setup-go/releases/tag/v5.5.0
         with:
           go-version: "1.25"
-      - name: Build the database (this takes some time)
-        run: |
-          # will fetch the latest build database from ghcr.io
-          go run ./cmd/devguard-cli/main.go vulndb sync
-
-      - name: Dump the PostgreSQL database
-        if: ${{ github.event.inputs.run_generate_snapshot == 'true' }}
-        # skip:checkov:CKV_SECRET_6
-        run: |
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM affected_components) TO STDOUT WITH DELIMITER ',' CSV HEADER" > affected_components.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM cve_affected_component) TO STDOUT WITH DELIMITER ',' CSV HEADER" > cve_affected_component.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM cves) TO STDOUT WITH DELIMITER ',' CSV HEADER" > cves.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM cwes) TO STDOUT WITH DELIMITER ',' CSV HEADER" > cwes.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM exploits) TO STDOUT WITH DELIMITER ',' CSV HEADER" > exploits.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM malicious_packages) TO STDOUT WITH DELIMITER ',' CSV HEADER" > malicious_packages.csv 
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM malicious_affected_components) TO STDOUT WITH DELIMITER ',' CSV HEADER" > malicious_affected_components.csv 
-          # PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM weaknesses) TO STDOUT WITH DELIMITER ',' CSV HEADER" > weaknesses.csv
-          PGPASSWORD=${{env.POSTGRES_PASSWORD}} psql -h localhost -U devguard devguard -c "COPY (SELECT * FROM cve_relationships) TO STDOUT WITH DELIMITER ',' CSV HEADER" > cve_relationships.csv
       - name: Export the diff csv files (this does not take some time)
-        if: ${{ github.event.inputs.run_generate_snapshot == 'false' || github.event.inputs.run_generate_snapshot == '' }}
         run: |
           # writes the difference from the db before and after the sync into csv files
           go run ./cmd/devguard-cli/main.go vulndb export
 
-      - name: install zip
-        run: sudo apt-get install zip
-
-      - name: Zip the CSV files
-        if: ${{ github.event.inputs.run_generate_snapshot == 'true' }}
-        run: zip vulndb.zip affected_components.csv  cve_affected_component.csv cves.csv cwes.csv exploits.csv malicious_packages.csv malicious_affected_components.csv cve_relationships.csv
-      - name: Zip the CSV files
-        if: ${{  github.event.inputs.run_generate_snapshot == 'false' || github.event.inputs.run_generate_snapshot == '' }}
-        run: zip -r vulndb.zip diffs-tmp
       - name: Install Cosign
         uses: sigstore/cosign-installer@faadad0cce49287aee09b3a48701e75088a2c6ad # v4.0.0
         with:
@@ -97,10 +64,14 @@ jobs:
       - name: Write signing key to disk
         run: echo "${{ secrets.COSIGN_PRIVATE_KEY }}" > cosign.key
 
-      - name: Sign the database zip file
+      - name: Sign each database file
         env:
           COSIGN_PASSWORD: ""
-        run: cosign import-key-pair --key cosign.key && cosign sign-blob --yes --key import-cosign.key vulndb.zip > vulndb.zip.sig
+        run: |
+          cosign import-key-pair --key cosign.key
+          cosign sign-blob --yes --key import-cosign.key diffOSVVulns.gob.zst > diffOSVVulns.gob.zst.sig
+          cosign sign-blob --yes --key import-cosign.key allOSVVulns.gob.zst > allOSVVulns.gob.zst.sig
+          cosign sign-blob --yes --key import-cosign.key integrity_checks.json > integrity_checks.json.sig
       - name: Login to GitHub Container Registry
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
         with:
@@ -113,20 +84,17 @@ jobs:
       - name: set the date
         run: echo "date="${{env.DATE}} >> "$GITHUB_ENV"
 
-      - name: Push the database ZIP file to GitHub Container Registry (vulndb/v1)
-        if: ${{  github.event.inputs.run_generate_snapshot == 'false' || github.event.inputs.run_generate_snapshot == '' }}
-        run: |
-          oras push ghcr.io/l3montree-dev/devguard/vulndb/v1:$date vulndb.zip
-
-      - name: Push the database ZIP file to GitHub Container Registry (snapshot)
-        if: ${{ github.event.inputs.run_generate_snapshot == 'true' }}
-        run: |
-          oras push ghcr.io/l3montree-dev/devguard/vulndb/v1:$date-snapshot vulndb.zip
-      - name: Push the signatures to the GitHub Container Registry
-        if: ${{  github.event.inputs.run_generate_snapshot == 'false' || github.event.inputs.run_generate_snapshot == '' }}
+      - name: Push the database files to GitHub Container Registry (snapshot)
         run: |
-          oras push ghcr.io/l3montree-dev/devguard/vulndb/v1:$date.sig vulndb.zip.sig
+          oras push ghcr.io/l3montree-dev/devguard/vulndb/osv-mirror:$date \
+            allOSVVulns.gob.zst \
+            diffOSVVulns.gob.zst \
+            integrity_checks.json
+          oras tag ghcr.io/l3montree-dev/devguard/vulndb/osv-mirror:$date latest
       - name: Push the signatures to the GitHub Container Registry (snapshot)
-        if: ${{ github.event.inputs.run_generate_snapshot == 'true' }}
         run: |
-          oras push ghcr.io/l3montree-dev/devguard/vulndb/v1:$date-snapshot.sig vulndb.zip.sig
+          oras push ghcr.io/l3montree-dev/devguard/vulndb/osv-mirror:$date.sig \
+            allOSVVulns.gob.zst.sig \
+            diffOSVVulns.gob.zst.sig \
+            integrity_checks.json.sig
+          oras tag ghcr.io/l3montree-dev/devguard/vulndb/osv-mirror:$date.sig latest.sig

.vscode/launch.json

@@ -42,16 +42,16 @@
             "program": "${workspaceRoot}/cmd/devguard/main.go",
             "args": [],
         },
-    {
-        "name": "Launch DepTreeWalk",
-        "type": "go",
-        "request": "launch",
-        "cwd": "${workspaceRoot}",
-        "mode": "auto",
-        "program": "${workspaceRoot}/cmd/devguard-cli/test",
-        "args": [
-        ],
-    },
+        {
+            "name": "Launch DepTreeWalk",
+            "type": "go",
+            "request": "launch",
+            "cwd": "${workspaceRoot}",
+            "mode": "auto",
+            "program": "${workspaceRoot}/cmd/devguard-cli/test",
+            "args": [
+            ],
+        },
         {
             "name": "Launch Policy eval",
             "type": "go",
@@ -105,6 +105,30 @@
                 "epss"
             ]
         },
+        {
+            "name": "VulnDB Import",
+            "type": "go",
+            "request": "launch",
+            "cwd": "${workspaceRoot}",
+            "mode": "auto",
+            "program": "${workspaceRoot}/cmd/devguard-cli/main.go",
+            "args": [
+                "vulndb",
+                "import"
+            ]
+        },
+         {
+            "name": "VulnDB ImportRC",
+            "type": "go",
+            "request": "launch",
+            "cwd": "${workspaceRoot}",
+            "mode": "auto",
+            "program": "${workspaceRoot}/cmd/devguard-cli/main.go",
+            "args": [
+                "vulndb",
+                "importRC"
+            ]
+        },
         {
             "name": "Scanner attestations",
             "type": "go",

cmd/devguard-cli/commands/vulndb.go

@@ -31,9 +31,8 @@ func NewVulndbCommand() *cobra.Command {
 
 	vulndbCmd.AddCommand(newSyncCommand())
 	vulndbCmd.AddCommand(newImportCommand())
-	vulndbCmd.AddCommand(newExportIncrementalCommand())
+	vulndbCmd.AddCommand(newExportCommand())
 	vulndbCmd.AddCommand(newAliasMappingCommand())
-	vulndbCmd.AddCommand(newCleanupCommand())
 	return &vulndbCmd
 }