Created Function for OpenVEX ingestion and openvex fetching from github repo

Author: Dboy0ZDev

normalize/sbom_graph.go

@@ -27,6 +27,7 @@ import (
 
 	cdx "github.com/CycloneDX/cyclonedx-go"
 	"github.com/google/uuid"
+	ov "github.com/openvex/go-vex/pkg/vex"
 	"github.com/package-url/packageurl-go"
 )
 
@@ -141,6 +142,41 @@ func NewVexReport(report *cdx.BOM, source string) (*VexReport, error) {
 	}, nil
 }
 
+type VexReportOpenVEX struct {
+	Report *ov.VEX
+	Source string
+}
+
+func validateVexReportOpenVEX(report *ov.VEX) error {
+	if report.ID == "" {
+		return fmt.Errorf("invalid OpenVEX report: missing id")
+	}
+	if report.Context == "" {
+		return fmt.Errorf("invalid OpenVEX report: missing context")
+	}
+	if report.Author == "" {
+		return fmt.Errorf("invalid OpenVEX report: missing author")
+	}
+	if report.Timestamp.IsZero() {
+		return fmt.Errorf("invalid OpenVEX report: missing timestamp")
+	}
+	if report.Version == 0 {
+		return fmt.Errorf("invalid OpenVEX report: missing version")
+	}
+	return nil
+}
+
+func NewVexReportOpenVEX(report *ov.VEX, source string) (*VexReportOpenVEX, error) {
+	if err := validateVexReportOpenVEX(report); err != nil {
+		return nil, err
+	}
+
+	return &VexReportOpenVEX{
+		Report: report,
+		Source: source,
+	}, nil
+}
+
 func edgesToDepMap(edges map[string]map[string]struct{}) map[string][]string {
 	depMap := make(map[string][]string)
 	for parent, children := range edges {

services/scan_service.go

@@ -22,11 +22,14 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"net/url"
+	"path"
 	"slices"
 	"strings"
 	"time"
 
 	"github.com/CycloneDX/cyclonedx-go"
+	"github.com/google/go-github/v62/github"
 	"github.com/google/uuid"
 	"github.com/l3montree-dev/devguard/database/models"
 	databasetypes "github.com/l3montree-dev/devguard/database/types"
@@ -38,6 +41,7 @@ import (
 	"github.com/l3montree-dev/devguard/transformer"
 	"github.com/l3montree-dev/devguard/utils"
 	"github.com/l3montree-dev/devguard/vulndb/scan"
+	ov "github.com/openvex/go-vex/pkg/vex"
 	"github.com/package-url/packageurl-go"
 	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel/attribute"
@@ -63,6 +67,12 @@ type scanService struct {
 	utils.FireAndForgetSynchronizer
 }
 
+var newGitHubClient = func() *github.Client {
+	return github.NewClient(nil)
+}
+
+var downloadRawFileFn = DownloadRawFile
+
 var _ shared.ScanService = (*scanService)(nil)
 
 func NewScanService(
@@ -864,3 +874,97 @@ func (s *scanService) ScanSBOMWithoutSaving(ctx context.Context, bom *cyclonedx.
 		DependencyVulns: vulnDTOs,
 	}, nil
 }
+
+func (s *scanService) FetchOpenVexFromGitHub(ctx context.Context, targetUrl string) (vexReports []*normalize.VexReportOpenVEX, err error) {
+	client := newGitHubClient()
+	githubDomain := "https://github.com"
+	if !strings.HasPrefix(targetUrl, githubDomain) {
+		return nil, fmt.Errorf("invalid github repository url")
+	}
+	owner, repo, err := ParseGitHubURL(targetUrl)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine default branch
+	repository, _, err := client.Repositories.Get(ctx, owner, repo)
+	if err != nil {
+		return nil, err
+	}
+	branch := repository.GetDefaultBranch()
+	if branch == "" {
+		branch = "main"
+	}
+
+	tree, _, err := client.Git.GetTree(
+		ctx,
+		owner,
+		repo,
+		branch,
+		true, // recursive
+	)
+	if err != nil {
+
+		return nil, err
+	}
+	for _, entry := range tree.Entries {
+		if entry.GetType() != "blob" {
+			continue
+		}
+		filePath := entry.GetPath()
+		filename := strings.ToLower(path.Base(filePath))
+		if !strings.HasSuffix(filename, ".json") {
+			continue
+		}
+
+		content, err := downloadRawFileFn(
+			owner,
+			repo,
+			branch,
+			filePath,
+		)
+		if err != nil {
+			slog.Info("download of openVEX failed", "err", err)
+			continue
+		}
+		var openVEX ov.VEX
+		err = json.Unmarshal(content, &openVEX)
+		if err != nil {
+			slog.Info("could not unmarshal openVEX failed", "err", err)
+			continue
+		}
+
+		vexReports = append(vexReports, &normalize.VexReportOpenVEX{
+			Report: &openVEX,
+			Source: targetUrl,
+		})
+	}
+	return vexReports, nil
+}
+
+func ParseGitHubURL(rawURL string) (owner string, repo string, err error) {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return "", "", err
+	}
+	parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+	return parts[0], parts[1], nil
+}
+
+func DownloadRawFile(owner, repo, branch, filePath string) ([]byte, error) {
+
+	rawURL := fmt.Sprintf(
+		"https://raw.githubusercontent.com/%s/%s/%s/%s",
+		owner,
+		repo,
+		branch,
+		filePath,
+	)
+	resp, err := http.Get(rawURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return io.ReadAll(resp.Body)
+
+}

services/scan_service_test.go

@@ -16,10 +16,14 @@ package services
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"testing"
+	"time"
 
+	"github.com/google/go-github/v62/github"
 	"github.com/google/uuid"
 	"github.com/l3montree-dev/devguard/database/models"
 	"github.com/l3montree-dev/devguard/dtos"
@@ -291,3 +295,149 @@ func TestFetchSbomsFromUpstream_PassesURLNotRef(t *testing.T) {
 		assert.Equal(t, sbomURL, invalidURLs[0].URL)
 	})
 }
+
+func TestFetchOpenVexFromGitHub(t *testing.T) {
+	originalNewGitHubClient := newGitHubClient
+	originalDownloadRawFileFn := downloadRawFileFn
+	t.Cleanup(func() {
+		newGitHubClient = originalNewGitHubClient
+		downloadRawFileFn = originalDownloadRawFileFn
+	})
+
+	t.Run("should fetch openvex reports from json files in the repository", func(t *testing.T) {
+		mockGitHub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case r.Method == http.MethodGet && r.URL.Path == "/repos/octo-org/openvex-repo":
+				_, _ = w.Write([]byte(`{"default_branch":"main"}`))
+			case r.Method == http.MethodGet && r.URL.Path == "/repos/octo-org/openvex-repo/git/trees/main":
+				if got := r.URL.Query().Get("recursive"); got != "1" {
+					t.Fatalf("expected recursive=1, got %q", got)
+				}
+				_, _ = w.Write([]byte(`{"tree":[{"path":"reports/openvex.json","type":"blob"},{"path":"README.md","type":"blob"}]}`))
+			default:
+				t.Fatalf("unexpected github api request: %s %s", r.Method, r.URL.String())
+			}
+		}))
+		defer mockGitHub.Close()
+
+		newGitHubClient = func() *github.Client {
+			client := github.NewClient(mockGitHub.Client())
+			baseURL, err := url.Parse(mockGitHub.URL + "/")
+			if err != nil {
+				t.Fatalf("failed to parse mock github url: %v", err)
+			}
+			client.BaseURL = baseURL
+			client.UploadURL = baseURL
+			return client
+		}
+
+		calls := 0
+		downloadRawFileFn = func(owner, repo, branch, filePath string) ([]byte, error) {
+			calls++
+			assert.Equal(t, "octo-org", owner)
+			assert.Equal(t, "openvex-repo", repo)
+			assert.Equal(t, "main", branch)
+			assert.Equal(t, "reports/openvex.json", filePath)
+
+			ts := time.Date(2026, time.May, 20, 12, 0, 0, 0, time.UTC)
+			payload := map[string]any{
+				"@context":   "https://openvex.dev/ns/v0.2.0",
+				"@id":        "openvex-1",
+				"author":     "test-author",
+				"timestamp":  ts,
+				"version":    1,
+				"statements": []any{},
+			}
+			return json.Marshal(payload)
+		}
+
+		service := &scanService{}
+		reports, err := service.FetchOpenVexFromGitHub(context.Background(), "https://github.com/octo-org/openvex-repo")
+		assert.NoError(t, err)
+		assert.Len(t, reports, 1)
+		assert.Equal(t, "https://github.com/octo-org/openvex-repo", reports[0].Source)
+		assert.Equal(t, "openvex-1", reports[0].Report.ID)
+		assert.Equal(t, "test-author", reports[0].Report.Author)
+		assert.Equal(t, 1, reports[0].Report.Version)
+		assert.Equal(t, 1, calls)
+	})
+
+	t.Run("should fetch multiple openvex reports from multiple json files", func(t *testing.T) {
+		mockGitHub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case r.Method == http.MethodGet && r.URL.Path == "/repos/octo-org/multi-vex-repo":
+				_, _ = w.Write([]byte(`{"default_branch":"develop"}`))
+			case r.Method == http.MethodGet && r.URL.Path == "/repos/octo-org/multi-vex-repo/git/trees/develop":
+				if got := r.URL.Query().Get("recursive"); got != "1" {
+					t.Fatalf("expected recursive=1, got %q", got)
+				}
+				_, _ = w.Write([]byte(`{"tree":[{"path":"vex/vex1.json","type":"blob"},{"path":"vex/vex2.json","type":"blob"},{"path":"README.md","type":"blob"}]}`))
+			default:
+				t.Fatalf("unexpected github api request: %s %s", r.Method, r.URL.String())
+			}
+		}))
+		defer mockGitHub.Close()
+
+		newGitHubClient = func() *github.Client {
+			client := github.NewClient(mockGitHub.Client())
+			baseURL, err := url.Parse(mockGitHub.URL + "/")
+			if err != nil {
+				t.Fatalf("failed to parse mock github url: %v", err)
+			}
+			client.BaseURL = baseURL
+			client.UploadURL = baseURL
+			return client
+		}
+
+		calls := 0
+		downloadRawFileFn = func(owner, repo, branch, filePath string) ([]byte, error) {
+			calls++
+			assert.Equal(t, "octo-org", owner)
+			assert.Equal(t, "multi-vex-repo", repo)
+			assert.Equal(t, "develop", branch)
+
+			ts := time.Date(2026, time.May, 20, 12, 0, 0, 0, time.UTC)
+			var id, author string
+
+			if filePath == "vex/vex1.json" {
+				id = "openvex-first"
+				author = "author-one"
+			} else if filePath == "vex/vex2.json" {
+				id = "openvex-second"
+				author = "author-two"
+			} else {
+				t.Fatalf("unexpected file path: %s", filePath)
+			}
+
+			payload := map[string]any{
+				"@context":   "https://openvex.dev/ns/v0.2.0",
+				"@id":        id,
+				"author":     author,
+				"timestamp":  ts,
+				"version":    1,
+				"statements": []any{},
+			}
+			return json.Marshal(payload)
+		}
+
+		service := &scanService{}
+		reports, err := service.FetchOpenVexFromGitHub(context.Background(), "https://github.com/octo-org/multi-vex-repo")
+		assert.NoError(t, err)
+		assert.Len(t, reports, 2)
+		assert.Equal(t, "https://github.com/octo-org/multi-vex-repo", reports[0].Source)
+		assert.Equal(t, "https://github.com/octo-org/multi-vex-repo", reports[1].Source)
+		assert.Equal(t, "openvex-first", reports[0].Report.ID)
+		assert.Equal(t, "openvex-second", reports[1].Report.ID)
+		assert.Equal(t, "author-one", reports[0].Report.Author)
+		assert.Equal(t, "author-two", reports[1].Report.Author)
+		assert.Equal(t, 2, calls)
+	})
+
+	t.Run("should reject non github urls", func(t *testing.T) {
+		service := &scanService{}
+		reports, err := service.FetchOpenVexFromGitHub(context.Background(), "https://example.com/repo")
+		assert.Error(t, err)
+		assert.Nil(t, reports)
+		assert.Contains(t, err.Error(), "invalid github repository url")
+	})
+}