feat: Create SourceRepository interface for Go (#4731)

This is part of the Go rewrite of the Importer, but I thought I'd split
up this PR to get some opinions on the database structure.

Adding a `SourceRepository` abstraction, and the bindings for the
Datastore object to allow the source repos to be read from Datastore.
There's an abstraction layer to separate out the business logic
(`internal/models`) from the Datastore API
(`internal/database/datastore`) which should make migrating off of
Datastore easier in the future.
I've moved & aliased the existing datastore models in `osv/models` into
`internal/database/datastore`, and will eventually try abstract away the
datastore usage from the exporter & relations computation.

Author: michaelkedar

…models/internal/validate/run_validate.sh …astore/internal/validate/run_validate.shgo/osv/models/internal/validate/run_validate.sh renamed to go/internal/database/datastore/internal/validate/run_validate.sh go/osv/models/internal/validate/run_validate.sh renamed to go/internal/database/datastore/internal/validate/run_validate.sh

@@ -9,7 +9,8 @@ import (
 	"time"
 
 	"cloud.google.com/go/datastore"
-	"github.com/google/osv.dev/go/osv/models"
+	db "github.com/google/osv.dev/go/internal/database/datastore"
+	"github.com/google/osv.dev/go/internal/models"
 )
 
 func main() {
@@ -30,65 +31,73 @@ func main() {
 func readRecords(ctx context.Context, client *datastore.Client) {
 	fmt.Println("(Go) Getting Vulnerability")
 	key := datastore.NameKey("Vulnerability", "CVE-123-456", nil)
-	var vulnerability models.Vulnerability
+	var vulnerability db.Vulnerability
 	if err := client.Get(ctx, key, &vulnerability); err != nil {
 		fmt.Printf("(Go) Failed getting Vulnerability: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting AliasGroup")
 	key = datastore.NameKey("AliasGroup", "1", nil)
-	var aliasGroup models.AliasGroup
+	var aliasGroup db.AliasGroup
 	if err := client.Get(ctx, key, &aliasGroup); err != nil {
 		fmt.Printf("(Go) Failed getting AliasGroup: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting AliasAllowListEntry")
 	key = datastore.NameKey("AliasAllowListEntry", "1", nil)
-	var aliasAllowListEntry models.AliasAllowListEntry
+	var aliasAllowListEntry db.AliasAllowListEntry
 	if err := client.Get(ctx, key, &aliasAllowListEntry); err != nil {
 		fmt.Printf("(Go) Failed getting AliasAllowListEntry: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting AliasDenyListEntry")
 	key = datastore.NameKey("AliasDenyListEntry", "1", nil)
-	var aliasDenyListEntry models.AliasDenyListEntry
+	var aliasDenyListEntry db.AliasDenyListEntry
 	if err := client.Get(ctx, key, &aliasDenyListEntry); err != nil {
 		fmt.Printf("(Go) Failed getting AliasDenyListEntry: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting UpstreamGroup")
 	key = datastore.NameKey("UpstreamGroup", "1", nil)
-	var upstreamGroup models.UpstreamGroup
+	var upstreamGroup db.UpstreamGroup
 	if err := client.Get(ctx, key, &upstreamGroup); err != nil {
 		fmt.Printf("(Go) Failed getting UpstreamGroup: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting ListedVulnerability")
 	key = datastore.NameKey("ListedVulnerability", "CVE-123-456", nil)
-	var listedVulnerability models.ListedVulnerability
+	var listedVulnerability db.ListedVulnerability
 	if err := client.Get(ctx, key, &listedVulnerability); err != nil {
 		fmt.Printf("(Go) Failed getting ListedVulnerability: %v\n", err)
 		os.Exit(1)
 	}
 
 	fmt.Println("(Go) Getting RelatedGroup")
 	key = datastore.NameKey("RelatedGroup", "CVE-123-456", nil)
-	var relatedGroup models.RelatedGroup
+	var relatedGroup db.RelatedGroup
 	if err := client.Get(ctx, key, &relatedGroup); err != nil {
 		fmt.Printf("(Go) Failed getting RelatedGroup: %v\n", err)
 		os.Exit(1)
 	}
+
+	fmt.Println("(Go) Getting SourceRepository")
+	key = datastore.NameKey("SourceRepository", "oss-fuzz", nil)
+	var sourceRepo db.SourceRepository
+	if err := client.Get(ctx, key, &sourceRepo); err != nil {
+		fmt.Printf("(Go) Failed getting SourceRepository: %v\n", err)
+		os.Exit(1)
+	}
 }
 
 func writeRecords(ctx context.Context, client *datastore.Client) {
 	fmt.Println("(Go) Writing Vulnerability")
 	key := datastore.NameKey("Vulnerability", "CVE-987-654", nil)
-	vulnerability := models.Vulnerability{
+	vulnerability := db.Vulnerability{
 		SourceID:    "test:path/to/CVE-987-654",
 		Modified:    time.Date(2025, time.December, 31, 23, 59, 59, 0, time.UTC),
 		IsWithdrawn: false,
@@ -104,7 +113,7 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing AliasGroup")
 	key = datastore.NameKey("AliasGroup", "2", nil)
-	aliasGroup := models.AliasGroup{
+	aliasGroup := db.AliasGroup{
 		VulnIDs:  []string{"A-1", "B-1", "C-1"},
 		Modified: time.Date(2025, time.January, 1, 1, 1, 1, 1, time.UTC),
 	}
@@ -115,7 +124,7 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing AliasAllowListEntry")
 	key = datastore.NameKey("AliasAllowListEntry", "2", nil)
-	aliasAllowListEntry := models.AliasAllowListEntry{
+	aliasAllowListEntry := db.AliasAllowListEntry{
 		VulnID: "IS-GOOD",
 	}
 	if _, err := client.Put(ctx, key, &aliasAllowListEntry); err != nil {
@@ -125,7 +134,7 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing AliasDenyListEntry")
 	key = datastore.NameKey("AliasDenyListEntry", "2", nil)
-	aliasDenyListEntry := models.AliasDenyListEntry{
+	aliasDenyListEntry := db.AliasDenyListEntry{
 		VulnID: "IS-BAD",
 	}
 	if _, err := client.Put(ctx, key, &aliasDenyListEntry); err != nil {
@@ -135,7 +144,7 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing UpstreamGroup")
 	key = datastore.NameKey("UpstreamGroup", "2", nil)
-	upstreamGroup := models.UpstreamGroup{
+	upstreamGroup := db.UpstreamGroup{
 		UpstreamIDs:       []string{"U-1", "U-2"},
 		Modified:          time.Date(2025, time.January, 1, 1, 1, 1, 1, time.UTC),
 		UpstreamHierarchy: []byte(`{"A": ["B"]}`),
@@ -147,13 +156,13 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing ListedVulnerability")
 	key = datastore.NameKey("ListedVulnerability", "CVE-987-654", nil)
-	listedVulnerability := models.ListedVulnerability{
+	listedVulnerability := db.ListedVulnerability{
 		Published:  time.Date(2025, time.December, 31, 23, 59, 59, 0, time.UTC),
 		Ecosystems: []string{"Go", "PyPI"},
 		Packages:   []string{"stdlib", "requests"},
 		Summary:    "A vulnerability",
 		IsFixed:    true,
-		Severities: []models.Severity{
+		Severities: []db.Severity{
 			{Type: "CVSS_V3", Score: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"},
 		},
 		AutocompleteTags: []string{"cve-987-654", "stdlib", "requests"},
@@ -166,12 +175,46 @@ func writeRecords(ctx context.Context, client *datastore.Client) {
 
 	fmt.Println("(Go) Writing RelatedGroup")
 	key = datastore.NameKey("RelatedGroup", "CVE-987-654", nil)
-	relatedGroup := models.RelatedGroup{
+	relatedGroup := db.RelatedGroup{
 		RelatedIDs: []string{"R-1", "R-2"},
 		Modified:   time.Date(2025, time.January, 1, 1, 1, 1, 1, time.UTC),
 	}
 	if _, err := client.Put(ctx, key, &relatedGroup); err != nil {
 		fmt.Printf("(Go) Failed writing RelatedGroup %v: %v\n", key, err)
 		os.Exit(1)
 	}
+
+	fmt.Println("(Go) Writing SourceRepository")
+	key = datastore.NameKey("SourceRepository", "go-source", nil)
+	lastUpdate := time.Date(2025, time.February, 1, 10, 0, 0, 0, time.UTC)
+	goSourceRepo := db.SourceRepository{
+		Type:                    models.SourceRepositoryTypeBucket,
+		Name:                    "go-source",
+		RepoURL:                 "https://example.com/go-source",
+		RepoUsername:            "user",
+		RepoBranch:              "master",
+		RESTAPIURL:              "http://localhost:8080/",
+		Bucket:                  "osv-test-bucket",
+		DirectoryPath:           "osv",
+		LastSyncedHash:          "zyxwvutsrqponmlkjihgfedcba",
+		LastUpdateDate:          &lastUpdate,
+		IgnorePatterns:          []string{"ignore", "pattern"},
+		Editable:                false,
+		Extension:               ".yaml",
+		KeyPath:                 "key",
+		IgnoreGit:               false,
+		DetectCherrypicks:       true,
+		ConsiderAllBranches:     true,
+		VersionsFromRepo:        true,
+		IgnoreLastImportTime:    true,
+		IgnoreDeletionThreshold: true,
+		Link:                    "https://example.com/go-source",
+		HumanLink:               "https://example.com/go-source/human",
+		DBPrefix:                []string{"GO-TEST", "GO-2-TEST"},
+		StrictValidation:        true,
+	}
+	if _, err := client.Put(ctx, key, &goSourceRepo); err != nil {
+		fmt.Printf("(Go) Failed writing SourceRepository %v: %v\n", key, err)
+		os.Exit(1)
+	}
 }

…osv/models/internal/validate/validate.go …/datastore/internal/validate/validate.gogo/osv/models/internal/validate/validate.go renamed to go/internal/database/datastore/internal/validate/validate.go go/osv/models/internal/validate/validate.go renamed to go/internal/database/datastore/internal/validate/validate.go

@@ -21,7 +21,7 @@
 import osv.tests
 from osv import Vulnerability, AliasGroup, AliasAllowListEntry, \
     AliasDenyListEntry, ListedVulnerability, Severity, UpstreamGroup, \
-    RelatedGroup
+    RelatedGroup, SourceRepository, SourceRepositoryType
 
 
 def main() -> int:
@@ -91,6 +91,36 @@ def main() -> int:
       modified=datetime.datetime(2025, 6, 7, 8, 9, 10, tzinfo=datetime.UTC),
   ).put()
 
+  print('(Python) Putting SourceRepository')
+  SourceRepository(
+      id='oss-fuzz',
+      type=SourceRepositoryType.GIT,
+      name='oss-fuzz',
+      repo_url='https://github.com/google/oss-fuzz',
+      repo_username='user',
+      repo_branch='master',
+      rest_api_url='http://127.0.0.1/',
+      bucket='bucket',
+      directory_path='vulns',
+      last_synced_hash='abcdef',
+      last_update_date=datetime.datetime(
+          2025, 1, 1, 12, 0, 0, tzinfo=datetime.UTC),
+      ignore_patterns=['.*\\.md', 'test/.*'],
+      editable=True,
+      extension='.json',
+      key_path='vulnerability',
+      ignore_git=False,
+      detect_cherrypicks=True,
+      consider_all_branches=False,
+      versions_from_repo=True,
+      ignore_last_import_time=True,
+      ignore_deletion_threshold=True,
+      link='https://github.com/google/oss-fuzz/blob/master/',
+      human_link='https://github.com/google/oss-fuzz/tree/master/',
+      db_prefix=['OSS-FUZZ', 'OTHER'],
+      strict_validation=True,
+  ).put()
+
   # Run Go program to read the Python-created entities in Go.
   # And write Go entities.
   result = subprocess.run(['go', 'run', './validate.go'], check=False, cwd='.')
@@ -119,11 +149,15 @@ def main() -> int:
   print('(Python) Getting RelatedGroup')
   if RelatedGroup.get_by_id('CVE-987-654') is None:
     return 1
+  print('(Python) Getting SourceRepository')
+  if SourceRepository.get_by_id('go-source') is None:
+    return 1
 
   return 0
 
 
 if __name__ == '__main__':
   with osv.tests.datastore_emulator(), ndb.Client().context():
+    print('Datastore emulator running')
     ret = main()
   sys.exit(ret)

…osv/models/internal/validate/validate.py …/datastore/internal/validate/validate.pygo/osv/models/internal/validate/validate.py renamed to go/internal/database/datastore/internal/validate/validate.py go/osv/models/internal/validate/validate.py renamed to go/internal/database/datastore/internal/validate/validate.py

@@ -0,0 +1,105 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package datastore contains definitions of Datastore entities for OSV.dev.
+package datastore
+
+import (
+	"time"
+
+	"cloud.google.com/go/datastore"
+	"github.com/google/osv.dev/go/internal/models"
+)
+
+type Vulnerability struct {
+	Key         *datastore.Key `datastore:"__key__"`
+	SourceID    string         `datastore:"source_id"`
+	Modified    time.Time      `datastore:"modified"`
+	IsWithdrawn bool           `datastore:"is_withdrawn"`
+	ModifiedRaw time.Time      `datastore:"modified_raw"`
+	AliasRaw    []string       `datastore:"alias_raw"`
+	RelatedRaw  []string       `datastore:"related_raw"`
+	UpstreamRaw []string       `datastore:"upstream_raw"`
+}
+
+type AliasGroup struct {
+	VulnIDs  []string  `datastore:"bug_ids"`
+	Modified time.Time `datastore:"last_modified"`
+}
+
+type UpstreamGroup struct {
+	Key               *datastore.Key `datastore:"__key__"`
+	VulnID            string         `datastore:"db_id"`
+	UpstreamIDs       []string       `datastore:"upstream_ids"`
+	Modified          time.Time      `datastore:"last_modified"`
+	UpstreamHierarchy []byte         `datastore:"upstream_hierarchy,noindex"`
+}
+
+type RelatedGroup struct {
+	Key        *datastore.Key `datastore:"__key__"`
+	RelatedIDs []string       `datastore:"related_ids"`
+	Modified   time.Time      `datastore:"modified"`
+}
+
+type AliasAllowListEntry struct {
+	VulnID string `datastore:"bug_id"`
+}
+
+type AliasDenyListEntry struct {
+	VulnID string `datastore:"bug_id"`
+}
+
+type Severity struct {
+	Type  string `datastore:"type"`
+	Score string `datastore:"score"`
+}
+
+type ListedVulnerability struct {
+	Key              *datastore.Key `datastore:"__key__"`
+	Published        time.Time      `datastore:"published"`
+	Ecosystems       []string       `datastore:"ecosystems"`
+	Packages         []string       `datastore:"packages,noindex"`
+	Summary          string         `datastore:"summary,noindex"`
+	IsFixed          bool           `datastore:"is_fixed,noindex"`
+	Severities       []Severity     `datastore:"severities"`
+	AutocompleteTags []string       `datastore:"autocomplete_tags"`
+	SearchIndices    []string       `datastore:"search_indices"`
+}
+
+type SourceRepository struct {
+	Type                    models.SourceRepositoryType `datastore:"type"`
+	Name                    string                      `datastore:"name"`
+	RepoURL                 string                      `datastore:"repo_url"`
+	RepoUsername            string                      `datastore:"repo_username"`
+	RepoBranch              string                      `datastore:"repo_branch"`
+	RESTAPIURL              string                      `datastore:"rest_api_url"`
+	Bucket                  string                      `datastore:"bucket"`
+	DirectoryPath           string                      `datastore:"directory_path"`
+	LastSyncedHash          string                      `datastore:"last_synced_hash"`
+	LastUpdateDate          *time.Time                  `datastore:"last_update_date"`
+	IgnorePatterns          []string                    `datastore:"ignore_patterns"`
+	Editable                bool                        `datastore:"editable"`
+	Extension               string                      `datastore:"extension"`
+	KeyPath                 string                      `datastore:"key_path"`
+	IgnoreGit               bool                        `datastore:"ignore_git"`
+	DetectCherrypicks       bool                        `datastore:"detect_cherrypicks"`
+	ConsiderAllBranches     bool                        `datastore:"consider_all_branches"`
+	VersionsFromRepo        bool                        `datastore:"versions_from_repo"`
+	IgnoreLastImportTime    bool                        `datastore:"ignore_last_import_time"`
+	IgnoreDeletionThreshold bool                        `datastore:"ignore_deletion_threshold"`
+	Link                    string                      `datastore:"link"`
+	HumanLink               string                      `datastore:"human_link"`
+	DBPrefix                []string                    `datastore:"db_prefix"`
+	StrictValidation        bool                        `datastore:"strict_validation"`
+}