feat(nvd): Add functions to help monitor performance - ConductAnalysis and CpuProfile (#4817)

Added CPU profiling to nvd conversion so we can easily check what's
taking the most time, and ConductAnalysis, which runs through all of the
outputted metrics files after conversion runs and produces a CSV file on
the outcomes.

Author: jess-lowe

vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go

@@ -9,9 +9,13 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"path/filepath"
+	"regexp"
+	"runtime/pprof"
 	"slices"
 	"sync"
 
+	"github.com/google/osv/vulnfeeds/conversion"
 	"github.com/google/osv/vulnfeeds/conversion/nvd"
 	"github.com/google/osv/vulnfeeds/cves"
 	"github.com/google/osv/vulnfeeds/git"
@@ -25,6 +29,8 @@ var (
 	outDir              = flag.String("out-dir", "", "Path to output results.")
 	outFormat           = flag.String("out-format", "OSV", "Format to output {OSV,PackageInfo}")
 	workers             = flag.Int("workers", 30, "The number of concurrent workers to use for processing CVEs.")
+	outputMetrics       = flag.Bool("output-metrics", true, "If true, output the metrics information about the conversion")
+	cpuProfile          = flag.String("cpuprofile", "", "Path to write cpu profile to file (default = no output)")
 )
 
 func loadCPEDictionary(productToRepo *cves.VPRepoCache, f string) error {
@@ -49,6 +55,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	if *cpuProfile != "" {
+		f, err := os.Create(*cpuProfile)
+		if err != nil {
+			logger.Fatal("could not create CPU profile: ", slog.Any("err", err))
+		}
+		defer f.Close()
+		if err := pprof.StartCPUProfile(f); err != nil {
+			logger.Fatal("could not start CPU profile: ", slog.Any("err", err))
+		}
+		defer pprof.StopCPUProfile()
+	}
+
 	logger.InitGlobalLogger()
 
 	data, err := os.ReadFile(*jsonPath)
@@ -88,6 +106,20 @@ func main() {
 	close(jobs)
 	wg.Wait()
 	logger.Info("NVD Conversion run complete")
+
+	// Conduct analysis on the outcome of the converted files and output to a csv
+	if *outputMetrics {
+		// Try to extract year from path, otherwise use "xxxx" filler
+		filename := filepath.Base(*jsonPath)
+		re := regexp.MustCompile(`nvdcve-2\.0-([0-9]{4})\.json`)
+		matches := re.FindStringSubmatch(filename)
+		if len(matches) >= 2 {
+			year := matches[1]
+			conversion.ConductAnalysis(year, *outDir)
+		} else {
+			conversion.ConductAnalysis("xxxx", *outDir)
+		}
+	}
 }
 
 func processCVE(cve models.NVDCVE, vpRepoCache *cves.VPRepoCache, repoTagsCache *git.RepoTagsCache) models.ConversionOutcome {

vulnfeeds/conversion/common.go

@@ -3,13 +3,16 @@
 package conversion
 
 import (
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/google/osv/vulnfeeds/models"
 	"github.com/google/osv/vulnfeeds/utility/logger"
@@ -69,6 +72,60 @@ func DeduplicateRefs(refs []models.Reference) []models.Reference {
 	return refs
 }
 
+// ConductAnalysis conducts an analysis of the conversion results after completion by reading
+// all of the .metrics.json files and extracting conversion outcomes.
+func ConductAnalysis(year string, dir string) {
+	// get the current time in minutes
+	currentTime := time.Now().Format("2006-01-02T15:04")
+	outcomesCSV := "nvd-conversion-outcomes-" + year + "-" + currentTime + ".csv"
+	csvFile, err := os.Create(filepath.Join(dir, outcomesCSV))
+	if err != nil {
+		logger.Fatal("Failed to create analysis CSV file", slog.Any("err", err))
+	}
+	defer csvFile.Close()
+
+	csvWriter := csv.NewWriter(csvFile)
+	defer csvWriter.Flush()
+
+	header := []string{"CVEID", "Outcome"}
+	if err := csvWriter.Write(header); err != nil {
+		logger.Fatal("Failed to write header to CSV", slog.Any("err", err))
+	}
+
+	err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && strings.HasSuffix(d.Name(), ".metrics.json") {
+			data, err := os.ReadFile(path)
+			if err != nil {
+				logger.Warn("Failed to read metrics file", slog.String("path", path), slog.Any("err", err))
+				return nil // Continue
+			}
+
+			var metrics models.ConversionMetrics
+			if err := json.Unmarshal(data, &metrics); err != nil {
+				logger.Warn("Failed to unmarshal metrics JSON", slog.String("path", path), slog.Any("err", err))
+				return nil // Continue
+			}
+
+			record := []string{
+				string(metrics.CVEID),
+				metrics.Outcome.String(),
+			}
+			if err := csvWriter.Write(record); err != nil {
+				logger.Warn("Failed to write record to CSV", slog.String("cve", string(metrics.CVEID)), slog.Any("err", err))
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		logger.Error("Failed to walk directory for analysis", slog.Any("err", err))
+	}
+}
+
 // CreateMetricsFile creates the initial file for the metrics record.
 func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) {
 	metricsFile := filepath.Join(vulnDir, string(id)+".metrics"+models.Extension)