feat: support new smart db

Author: G-Rath

.golangci.yaml

@@ -23,6 +23,7 @@ linters:
     - varnamelen      # maybe later
     - wsl             # disagree with, for now
     - wsl_v5          # disagree with, for now
+    - wrapcheck
   settings:
     depguard:
       rules:

main.go

@@ -33,10 +33,15 @@ func makeAPIDBConfig() database.Config {
 	}
 }
 
-func makeEcosystemDBConfig(ecosystem internal.Ecosystem) database.Config {
+func makeEcosystemDBConfig(ecosystem internal.Ecosystem, beSmart bool) database.Config {
+	typ := "zip"
+	if beSmart {
+		typ = "smart"
+	}
+
 	return database.Config{
 		Name: string(ecosystem),
-		Type: "zip",
+		Type: typ,
 		URL:  fmt.Sprintf("https://osv-vulnerabilities.storage.googleapis.com/%s/all.zip", ecosystem),
 	}
 }
@@ -158,6 +163,15 @@ func describeDB(db database.DB) string {
 	switch tt := db.(type) {
 	case *database.APIDB:
 		return "using batches of " + color.YellowString("%d", tt.BatchSize)
+	case *database.SmartDB:
+		count := tt.VulnerabilitiesCount
+
+		return fmt.Sprintf(
+			"%s %s, including withdrawn - last updated %s",
+			color.YellowString("%d", count),
+			reporter.Form(count, "vulnerability", "vulnerabilities"),
+			tt.UpdatedAt,
+		)
 	case *database.ZipDB:
 		count := tt.VulnerabilitiesCount
 
@@ -372,6 +386,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
 	removeConfigDatabases bool,
 	addDefaultAPIDatabase bool,
 	addEcosystemDatabases bool,
+	beSmart bool,
 ) {
 	for _, file := range files {
 		if file.err != nil {
@@ -391,7 +406,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
 			ecosystems := collectEcosystems([]lockfileAndConfigOrErr{file})
 
 			for _, ecosystem := range ecosystems {
-				extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem))
+				extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem, beSmart))
 			}
 		}
 
@@ -508,6 +523,7 @@ func run(args []string, stdout, stderr io.Writer) int {
 	useDatabases := cli.Bool("use-dbs", true, "Use the databases from osv.dev to check for known vulnerabilities")
 	useAPI := cli.Bool("use-api", false, "Use the osv.dev API to check for known vulnerabilities")
 	batchSize := cli.Int("batch-size", 1000, "The number of packages to include in each batch when using the api database")
+	beSmart := cli.Bool("be-smart", false, "")
 
 	cli.Var(&globalIgnores, "ignore", `ID of an OSV to ignore when determining exit codes.
 This flag can be passed multiple times to ignore different vulnerabilities`)
@@ -589,7 +605,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
 
 	files := readAllLockfiles(r, pathsToLocksWithParseAs, cli.Args(), loadLocalConfig, &config)
 
-	files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases)
+	files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases, *beSmart)
 
 	dbs, errored := loadDatabases(
 		r,

pkg/database/config.go

@@ -29,6 +29,8 @@ var ErrUnsupportedDatabaseType = errors.New("unsupported database source type")
 // Load initializes a new OSV database based on the given Config
 func Load(config Config, offline bool, batchSize int) (DB, error) {
 	switch config.Type {
+	case "smart":
+		return NewSmartDB(config, offline)
 	case "zip":
 		return NewZippedDB(config, offline)
 	case "api":

pkg/database/smart.go

@@ -0,0 +1,309 @@
+package database
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type SmartDB struct {
+	// memDB
+	DirDB
+
+	name             string
+	identifier       string
+	ArchiveURL       string
+	WorkingDirectory string
+	Offline          bool
+	UpdatedAt        string
+}
+
+func (db *SmartDB) Name() string       { return db.name }
+func (db *SmartDB) Identifier() string { return db.identifier }
+
+func (db *SmartDB) cachePath() string {
+	hash := sha256.Sum256([]byte(db.ArchiveURL))
+	fileName := fmt.Sprintf("osv-detector-%x-db", hash)
+
+	return filepath.Join(os.TempDir(), fileName)
+}
+
+func (db *SmartDB) cacheFile(name string, content []byte) error {
+	//nolint:gosec // being world readable is fine
+	return os.WriteFile(filepath.Join(db.cachePath(), name), content, 0644)
+}
+
+func (db *SmartDB) loadLastModified() (time.Time, error) {
+	b, err := os.ReadFile(filepath.Join(db.cachePath(), "last_modified"))
+
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	tim, err := time.Parse(time.RFC3339, string(b))
+
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	return tim, nil
+}
+
+func (db *SmartDB) updateLastModified(lastModified time.Time) error {
+	db.UpdatedAt = lastModified.Format(http.TimeFormat)
+
+	return db.cacheFile("last_modified", []byte(lastModified.Format(time.RFC3339)))
+}
+
+func (db *SmartDB) writeZipFile(zipFile *zip.File) error {
+	dst, err := os.OpenFile(filepath.Join(db.cachePath(), zipFile.Name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode())
+	if err != nil {
+		return err
+	}
+
+	defer dst.Close()
+
+	z, err := zipFile.Open()
+	if err != nil {
+		return err
+	}
+
+	defer z.Close()
+
+	_, err = io.Copy(dst, z)
+
+	return err
+}
+
+func (db *SmartDB) populateFromZip() error {
+	err := os.MkdirAll(db.cachePath(), 0744)
+
+	if err != nil {
+		return err
+	}
+
+	zdb := &ZipDB{
+		name:             db.name,
+		ArchiveURL:       db.ArchiveURL,
+		WorkingDirectory: db.WorkingDirectory,
+		Offline:          db.Offline,
+	}
+
+	body, err := zdb.fetchZip()
+
+	if err != nil {
+		return err
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		return fmt.Errorf("could not read OSV database archive: %w", err)
+	}
+
+	// Read each file from the archive and write it to the db directory
+	for _, zipFile := range zipReader.File {
+		if !strings.HasPrefix(zipFile.Name, db.WorkingDirectory) {
+			continue
+		}
+
+		if !strings.HasSuffix(zipFile.Name, ".json") {
+			continue
+		}
+
+		err = db.writeZipFile(zipFile)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type modifiedIDRow struct {
+	id       string
+	modified time.Time
+}
+
+func parseModifiedIDRow(columns []string) (*modifiedIDRow, error) {
+	modified, err := time.Parse(time.RFC3339, columns[0])
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &modifiedIDRow{id: columns[1], modified: modified}, nil
+}
+
+func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) {
+	url := strings.TrimSuffix(db.ArchiveURL, "/all.zip") + "/modified_id.csv"
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%w (%s)", ErrUnexpectedStatusCode, resp.Status)
+	}
+
+	i := 0
+	r := csv.NewReader(resp.Body)
+
+	var rows []modifiedIDRow
+
+	for {
+		i++
+		record, err := r.Read()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%w", err)
+		}
+
+		row, err := parseModifiedIDRow(record)
+		if err != nil {
+			return nil, fmt.Errorf("row %d: %w", i, err)
+		}
+
+		// the modified ids are sorted in reverse chronological order so once we hit
+		// a row that was modified before our "since" time, we can stop completely
+		if row.modified.Before(since) {
+			break
+		}
+
+		rows = append(rows, *row)
+	}
+
+	return rows, nil
+}
+
+func (db *SmartDB) updateAdvisory(id string) error {
+	url := fmt.Sprintf("%s/%s.json", strings.TrimSuffix(db.ArchiveURL, "/all.zip"), id)
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%w (%s)", ErrUnexpectedStatusCode, resp.Status)
+	}
+
+	var body []byte
+
+	body, err = io.ReadAll(resp.Body)
+
+	if err != nil {
+		return err
+	}
+
+	content, err := json.Marshal(body)
+
+	if err == nil {
+		err = db.cacheFile(id+".json", content)
+	}
+
+	return err
+}
+
+func (db *SmartDB) updateModifiedAdvisories(since time.Time) error {
+	modifiedIDs, err := db.fetchModifiedIDs(since)
+
+	if err != nil {
+		return err
+	}
+
+	for _, row := range modifiedIDs {
+		// fmt.Printf("updating %s\n", row.id)
+		err = db.updateAdvisory(row.id)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// load fetches a zip archive of the OSV database and loads known vulnerabilities
+// from it (which are assumed to be in json files following the OSV spec).
+//
+// Internally, the archive is cached along with the date that it was fetched
+// so that a new version of the archive is only downloaded if it has been
+// modified, per HTTP caching standards.
+func (db *SmartDB) load() error {
+	lastModified, err := db.loadLastModified()
+
+	// (re)initialize the database using the zip file
+	if err != nil {
+		err = db.populateFromZip()
+
+		if err != nil {
+			return err
+		}
+
+		// update the last modified time to now
+		lastModified = time.Now()
+	}
+
+	// update any advisories that have changed since the last modified timestamp
+	err = db.updateModifiedAdvisories(lastModified)
+	if err != nil {
+		return err
+	}
+
+	if err = db.updateLastModified(lastModified); err != nil {
+		return err
+	}
+
+	db.DirDB = DirDB{
+		name:             db.name,
+		LocalPath:        "file:///" + db.cachePath(),
+		WorkingDirectory: "",
+		Offline:          db.Offline,
+	}
+
+	return db.DirDB.load()
+}
+
+func NewSmartDB(config Config, offline bool) (*SmartDB, error) {
+	db := &SmartDB{
+		name:             config.Name,
+		identifier:       config.Identifier(),
+		ArchiveURL:       config.URL,
+		WorkingDirectory: config.WorkingDirectory,
+		Offline:          offline,
+	}
+	if err := db.load(); err != nil {
+		return nil, fmt.Errorf("unable to fetch OSV database: %w", err)
+	}
+
+	return db, nil
+}