feat: Re-implement importer in Go (#4808)

Big re-implementation of the importer (and importer-deleter) into Go:
- Implemented go gitter getter
- Moved distinct logic (git/GCS/REST) into relevant files
- Removed some outdated things (e.g. public_logs_bucket)
- I've remove the writing of ImportFindings, because I think the linter
is currently also doing (and overwriting) these
- Parallelised processing of source repositories
- also parallelised downloading records and publishing to pubsub within
a source repository via a shared worker pool
- Made interfaces so that the code is testable.

I'd appreciate people reviewing this to see how
follow-able/self-documenting the code is, and point out where things may
be unclear.

Author: michaelkedar

Makefile

@@ -55,7 +55,7 @@ update-api-snapshots:
 	cd gcp/api && UPDATE_SNAPS=true ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json
 
 lint:
-	GOTOOLCHAIN=go1.25.7 $(run-cmd) tools/lint_and_format.sh
+	GOTOOLCHAIN=go1.26.0 $(run-cmd) tools/lint_and_format.sh
 
 build-osv-protos:
 	cd osv && $(run-cmd) python -m grpc_tools.protoc --python_out=. --mypy_out=. --proto_path=. --proto_path=osv-schema/proto vulnerability.proto importfinding.proto

deployment/build-and-stage.yaml

@@ -62,7 +62,7 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/worker-base']
   waitFor: ['build-worker-base', 'cloud-build-queue']
 
-# Build/push core worker/importer/alias images.
+# Build/push core worker/recoverer images.
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/worker:latest', '-t', 'gcr.io/oss-vdb/worker:$COMMIT_SHA', '-f', 'gcp/workers/worker/Dockerfile', '.']
   id: 'build-worker'
@@ -71,15 +71,6 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/worker']
   waitFor: ['build-worker', 'cloud-build-queue']
 
-- name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/importer:latest', '-t', 'gcr.io/oss-vdb/importer:$COMMIT_SHA', '.']
-  dir: 'gcp/workers/importer'
-  id: 'build-importer'
-  waitFor: ['build-worker']
-- name: gcr.io/cloud-builders/docker
-  args: ['push', '--all-tags', 'gcr.io/oss-vdb/importer']
-  waitFor: ['build-importer', 'cloud-build-queue']
-
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/recoverer:latest', '-t', 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA', '.']
   dir: 'gcp/workers/recoverer'
@@ -107,6 +98,20 @@ steps:
   waitFor: ['build-oss-fuzz-importer', 'cloud-build-queue']
 
 # Build/push go images
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'bash'
+  args: ['-c', 'docker pull gcr.io/oss-vdb/importer:latest || exit 0']
+  id: 'pull-importer'
+  waitFor: ['setup']
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/oss-vdb/importer:latest', '-t', 'gcr.io/oss-vdb/importer:$COMMIT_SHA', '-f', 'cmd/importer/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/importer:latest', '--pull', '.']
+  dir: 'go'
+  id: 'build-importer'
+  waitFor: ['pull-importer']
+- name: gcr.io/cloud-builders/docker
+  args: ['push', '--all-tags', 'gcr.io/oss-vdb/importer']
+  waitFor: ['build-importer', 'cloud-build-queue']
+
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: 'bash'
   args: ['-c', 'docker pull gcr.io/oss-vdb/exporter:latest || exit 0']

deployment/clouddeploy/gke-workers/base/importer.yaml

@@ -25,6 +25,10 @@ spec:
             env:
               - name: GITTER_HOST
                 value: http://gitter-service:8888
+              - name: IMPORT_TRACE_SAMPLE_RATE # for the overall importer trace
+                value: "1.0"
+              - name: TRACE_SAMPLE_RATE # for the individual vulnerability entries
+                value: "0.05"
             securityContext:
               privileged: true
             resources:

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/importer-deleter.yaml

@@ -17,5 +17,4 @@ spec:
             image: importer
             args:
               - --delete
-              - --delete_threshold_pct=2
-              - --public_log_bucket=osv-test-public-import-logs
+              - --delete-threshold-pct=2

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/importer.yaml

@@ -15,10 +15,7 @@ spec:
             - name: OSV_VULNERABILITIES_BUCKET
               value: osv-test-vulnerabilities
             args:
-              # TODO(michaelkedar): ssh secrets
-              # TODO(michaelkedar): single source of truth w/ terraform config
-              - "--public_log_bucket=osv-test-public-import-logs"
               # Note that with https://github.com/google/osv.dev/pull/2766
               # addition per-repository settings make this *really* take effect, see
               # https://github.com/google/osv.dev/pull/2837
-              - "--strict_validation"
+              - "--strict-validation"

deployment/clouddeploy/gke-workers/environments/oss-vdb/importer-deleter.yaml

@@ -17,5 +17,4 @@ spec:
             image: importer
             args:
               - --delete
-              - --delete_threshold_pct=2
-              - --public_log_bucket=osv-public-import-logs
+              - --delete-threshold-pct=2

deployment/clouddeploy/gke-workers/environments/oss-vdb/importer.yaml

@@ -14,20 +14,3 @@ spec:
               value: oss-vdb
             - name: OSV_VULNERABILITIES_BUCKET
               value: osv-vulnerabilities
-            args:
-              - "--ssh_key_public=/secrets/ssh.pub"
-              - "--ssh_key_private=/secrets/ssh"
-              - "--public_log_bucket=osv-public-import-logs"
-            volumeMounts:
-              - mountPath: "/secrets"
-                name: "secrets"
-          volumes:
-          - name: secrets
-            secret:
-              items:
-              - key: ssh
-                mode: 384
-                path: ssh
-              - key: ssh.pub
-                path: ssh.pub
-              secretName: secrets

go/cmd/exporter/exporter.go

@@ -97,19 +97,19 @@ func main() {
 
 	prevPrefix := ""
 MainLoop:
-	for path, err := range vulnClient.Objects(ctx, gcsProtoPrefix) {
+	for obj, err := range vulnClient.Objects(ctx, gcsProtoPrefix) {
 		if err != nil {
 			logger.FatalContext(ctx, "failed to list objects", slog.Any("err", err))
 		}
 		// Only log when we see a new ID prefix (i.e. roughly once per data source)
-		prefix := filepath.Base(path)
+		prefix := filepath.Base(obj.Name)
 		prefix, _, _ = strings.Cut(prefix, "-")
 		if prefix != prevPrefix {
-			logger.InfoContext(ctx, "iterating vulnerabilities", slog.String("now_at", path))
+			logger.InfoContext(ctx, "iterating vulnerabilities", slog.String("now_at", obj.Name))
 			prevPrefix = prefix
 		}
 		select {
-		case gcsPathToDownloaderCh <- path:
+		case gcsPathToDownloaderCh <- obj.Name:
 		case <-ctx.Done():
 			break MainLoop
 		}

go/cmd/generatesitemap/generatesitemap.go

@@ -180,19 +180,19 @@ func main() {
 
 func listObjects(ctx context.Context, client clients.CloudStorage, outCh chan<- string) error {
 	prevPrefix := ""
-	for name, err := range client.Objects(ctx, gcsProtoPrefix) {
+	for obj, err := range client.Objects(ctx, gcsProtoPrefix) {
 		if err != nil {
 			return err
 		}
 		// Only log when we see a new ID prefix (i.e. roughly once per data source)
-		prefix := filepath.Base(name)
+		prefix := filepath.Base(obj.Name)
 		prefix, _, _ = strings.Cut(prefix, "-")
 		if prefix != prevPrefix {
-			logger.InfoContext(ctx, "iterating vulnerabilities", slog.String("now_at", name))
+			logger.InfoContext(ctx, "iterating vulnerabilities", slog.String("now_at", obj.Name))
 			prevPrefix = prefix
 		}
 		select {
-		case outCh <- name:
+		case outCh <- obj.Name:
 		case <-ctx.Done():
 			return ctx.Err()
 		}

go/cmd/importer/Dockerfile

@@ -0,0 +1,34 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.26.0-alpine@sha256:d4c4845f5d60c6a974c6000ce58ae079328d03ab7f721a0734277e69905473e5 AS build
+
+WORKDIR /src
+
+COPY ./go.mod /src/go.mod
+COPY ./go.sum /src/go.sum
+RUN go mod download && go mod verify
+
+
+COPY ./ /src/
+RUN CGO_ENABLED=0 go build -o importer ./cmd/importer/
+
+FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659
+
+# Need to install the full tar package, to not use the busybox version, which doesn't have --zstd support.
+RUN apk add --no-cache git zstd tar
+
+COPY --from=build /src/importer /
+
+ENTRYPOINT ["/importer"]