Skip to content

Commit 06bcec1

Browse files
michaelkedarmichaelkedar
committed
formatting
1 parent 39a967c commit 06bcec1

16 files changed

Lines changed: 151 additions & 84 deletions

File tree

‎go/cmd/importer/main.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// Package main for importer synchronizes vulnerability data from various sources into the OSV ecosystem.
12
package main
23

34
import (
@@ -22,7 +23,7 @@ func main() {
2223
logger.InitGlobalLogger()
2324

2425
strictValidation := flag.Bool("strict-validation", false, "Fail to import entries that do not pass validation.")
25-
delete := flag.Bool("delete", false, "Bypass importing and propagate record deletions from source to Datastore")
26+
runDelete := flag.Bool("delete", false, "Bypass importing and propagate record deletions from source to Datastore")
2627
deleteThresholdPct := flag.Float64("delete-threshold-pct", 10.0, "More than this percent of records for a given source being deleted triggers an error")
2728
workDir := flag.String("work-dir", "/work", "Work directory for git repos")
2829
numWorkers := flag.Int("num-workers", 50, "Number of workers to use for importing")
@@ -68,7 +69,7 @@ func main() {
6869
}
6970
config.GCSProvider = clients.NewGCSStorageProvider(storageClient)
7071

71-
if *delete {
72+
if *runDelete {
7273
if err := importer.RunDeletions(context.Background(), config); err != nil {
7374
logger.Fatal("Importer-deleter failed", slog.Any("error", err))
7475
}

‎go/internal/database/datastore/vulnerability.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package datastore
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"iter"
78
"strings"
@@ -40,7 +41,7 @@ func (s *VulnerabilityStore) ListBySource(ctx context.Context, source string, sk
4041
for {
4142
var v Vulnerability
4243
key, err := it.Next(&v)
43-
if err == iterator.Done {
44+
if errors.Is(err, iterator.Done) {
4445
return
4546
}
4647
if err != nil {

‎go/internal/importer/bucket.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ func (b bucketSourceRecord) Open(ctx context.Context) (io.ReadCloser, error) {
3737
if err != nil {
3838
return nil, err
3939
}
40+
4041
return io.NopCloser(bytes.NewReader(data)), nil
4142
}
4243

@@ -157,7 +158,7 @@ func handleDeleteBucket(ctx context.Context, ch chan<- SourceRecord, config Conf
157158
}
158159

159160
// Get all non-withdrawn vulnerabilities in Datastore for this source
160-
var vulnsInDatastore []*models.VulnSourceRef
161+
vulnsInDatastore := make([]*models.VulnSourceRef, 0, len(objectsInBucket))
161162
for entry, err := range config.VulnerabilityStore.ListBySource(ctx, sourceRepo.Name, true) {
162163
if err != nil {
163164
return err
@@ -196,6 +197,7 @@ func handleDeleteBucket(ctx context.Context, ch chan<- SourceRecord, config Conf
196197
slog.Int("total", len(vulnsInDatastore)),
197198
slog.Float64("percentage", percentage),
198199
slog.Float64("threshold", threshold))
200+
199201
return errors.New("deletion threshold exceeded")
200202
}
201203

‎go/internal/importer/bucket_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func TestHandleImportBucket(t *testing.T) {
6969
}
7070

7171
mockStore := &mockSourceRepositoryStore{
72-
updates: make(map[string]interface{}),
72+
updates: make(map[string]any),
7373
}
7474

7575
config := Config{
@@ -97,7 +97,7 @@ func TestHandleImportBucket(t *testing.T) {
9797
}
9898

9999
close(ch)
100-
var records []bucketSourceRecord
100+
records := make([]bucketSourceRecord, 0, 10)
101101
for r := range ch {
102102
records = append(records, r.(bucketSourceRecord))
103103
}

‎go/internal/importer/deletion_test.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func TestHandleDeleteBucket(t *testing.T) {
5959
}
6060
close(workCh)
6161

62-
var records []SourceRecord
62+
records := make([]SourceRecord, 0, 10)
6363
for r := range workCh {
6464
records = append(records, r)
6565
}
@@ -134,8 +134,10 @@ func TestHandleDeleteREST(t *testing.T) {
134134
ctx := t.Context()
135135

136136
// Mock Server returns 1 remaining ID
137-
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
138-
w.Write([]byte(`[{"id": "STILL-THERE"}]`))
137+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
138+
if _, err := w.Write([]byte(`[{"id": "STILL-THERE"}]`)); err != nil {
139+
t.Fatalf("Failed to write response: %v", err)
140+
}
139141
}))
140142
defer ts.Close()
141143

@@ -174,7 +176,7 @@ func TestHandleDeleteREST(t *testing.T) {
174176
}
175177
close(workCh)
176178

177-
var records []SourceRecord
179+
records := make([]SourceRecord, 0, 10)
178180
for r := range workCh {
179181
records = append(records, r)
180182
}

‎go/internal/importer/filter.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ func negatedPrefix(regex string) (string, bool) {
2020
if strings.HasPrefix(regex, prefix) && strings.HasSuffix(regex, suffix) {
2121
return strings.TrimSuffix(strings.TrimPrefix(regex, prefix), suffix), true
2222
}
23+
2324
return "", false
2425
}
2526

@@ -39,10 +40,13 @@ func compileIgnorePatterns(sourceRepo *models.SourceRepository) []*regexp.Regexp
3940
slog.String("source", sourceRepo.Name),
4041
slog.String("pattern", pattern),
4142
slog.Any("error", err))
43+
4244
continue
4345
}
46+
4447
compiledIgnorePatterns = append(compiledIgnorePatterns, compiledPattern)
4548
}
49+
4650
return compiledIgnorePatterns
4751
}
4852

@@ -66,5 +70,6 @@ func shouldIgnore(name string, idPrefixes []string, ignorePatterns []*regexp.Reg
6670
return true
6771
}
6872
}
73+
6974
return false
7075
}

‎go/internal/importer/git.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ type gitSourceRecord struct {
3838

3939
var _ SourceRecord = gitSourceRecord{}
4040

41-
func (g gitSourceRecord) Open(ctx context.Context) (io.ReadCloser, error) {
41+
func (g gitSourceRecord) Open(_ context.Context) (io.ReadCloser, error) {
4242
if g.isDeleted {
4343
return nil, errors.New("cannot open a deleted record")
4444
}
@@ -57,6 +57,7 @@ func (g gitSourceRecord) Open(ctx context.Context) (io.ReadCloser, error) {
5757
if err != nil {
5858
return nil, err
5959
}
60+
6061
return io.NopCloser(bytes.NewReader(content)), nil
6162
}
6263

@@ -98,6 +99,7 @@ var repoGroup singleflight.Group
9899

99100
type sharedRepo struct {
100101
*git.Repository
102+
101103
mu *sync.Mutex
102104
}
103105

@@ -109,7 +111,7 @@ func handleImportGit(ctx context.Context, ch chan<- SourceRecord, config Config,
109111
slog.String("source", sourceRepo.Name), slog.String("url", sourceRepo.Git.URL))
110112

111113
compiledIgnorePatterns := compileIgnorePatterns(sourceRepo)
112-
repoInterface, err, _ := repoGroup.Do(sourceRepo.Git.URL, func() (interface{}, error) {
114+
repoInterface, err, _ := repoGroup.Do(sourceRepo.Git.URL, func() (any, error) {
113115
// Temporary migration from Python to Go
114116
// If the sha name of the repo doesn't exist, check if the source repo name exists from python.
115117
// If it does, move it and use it.
@@ -127,6 +129,7 @@ func handleImportGit(ctx context.Context, ch chan<- SourceRecord, config Config,
127129
if err != nil {
128130
return nil, err
129131
}
132+
130133
return sharedRepo{
131134
Repository: repo,
132135
mu: &sync.Mutex{},
@@ -166,6 +169,7 @@ func handleImportGit(ctx context.Context, ch chan<- SourceRecord, config Config,
166169
if shouldIgnore(path.Base(p), sourceRepo.IDPrefixes, compiledIgnorePatterns) {
167170
return ""
168171
}
172+
169173
return p
170174
}
171175
for _, fileChange := range changedFiles {
@@ -183,6 +187,7 @@ func handleImportGit(ctx context.Context, ch chan<- SourceRecord, config Config,
183187
isDeleted: true,
184188
strict: sourceRepo.Strictness,
185189
}
190+
186191
continue
187192
}
188193
// object created/modified - send to channel
@@ -206,6 +211,7 @@ func handleImportGit(ctx context.Context, ch chan<- SourceRecord, config Config,
206211
logger.Info("Finished importing git source repository",
207212
slog.String("source", sourceRepo.Name),
208213
slog.String("url", sourceRepo.Git.URL))
214+
209215
return nil
210216
}
211217

@@ -255,13 +261,14 @@ func changedFiles(ctx context.Context, repo sharedRepo, sourceRepo *models.Sourc
255261
if err != nil {
256262
return nil, nil, err
257263
}
258-
var changedFiles []fileChange
264+
changedFiles := make([]fileChange, 0, len(diff))
259265
for _, d := range diff {
260266
// Note: since we're doing child.Diff(parent), to/from are reversed from what you might expect.
261267
changedFiles = append(changedFiles, fileChange{
262268
from: d.To.Name,
263269
to: d.From.Name,
264270
})
265271
}
272+
266273
return changedFiles, currentCommit, nil
267274
}

‎go/internal/importer/git_test.go‎

Lines changed: 40 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,7 @@ import (
1515

1616
func TestGitSourceRecord_Open(t *testing.T) {
1717
// Setup a temporary git repo
18-
dir, err := os.MkdirTemp("", "osv-git-test-*")
19-
if err != nil {
20-
t.Fatalf("Failed to create temp dir: %v", err)
21-
}
22-
defer os.RemoveAll(dir)
23-
18+
dir := t.TempDir()
2419
repo, err := git.PlainInit(dir, false)
2520
if err != nil {
2621
t.Fatalf("Failed to init git repo: %v", err)
@@ -32,7 +27,7 @@ func TestGitSourceRecord_Open(t *testing.T) {
3227

3328
// Create a file
3429
filePath := filepath.Join(dir, "test.json")
35-
if err := os.WriteFile(filePath, []byte("data"), 0644); err != nil {
30+
if err := os.WriteFile(filePath, []byte("data"), 0600); err != nil {
3631
t.Fatalf("Failed to write file: %v", err)
3732
}
3833
if _, err := wt.Add("test.json"); err != nil {
@@ -79,12 +74,7 @@ func TestGitSourceRecord_Open(t *testing.T) {
7974

8075
func TestHandleImportGit(t *testing.T) {
8176
// Setup a temporary git repo acting as the remote source
82-
remoteDir, err := os.MkdirTemp("", "osv-git-remote-*")
83-
if err != nil {
84-
t.Fatalf("Failed to create temp dir: %v", err)
85-
}
86-
defer os.RemoveAll(remoteDir)
87-
77+
remoteDir := t.TempDir()
8878
remoteRepo, err := git.PlainInit(remoteDir, false)
8979
if err != nil {
9080
t.Fatalf("Failed to init remote repo: %v", err)
@@ -95,28 +85,43 @@ func TestHandleImportGit(t *testing.T) {
9585
}
9686

9787
// Initial commit: ignored file and old file
98-
os.WriteFile(filepath.Join(remoteDir, "ignore.json"), []byte("{}"), 0644)
99-
os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte("{}"), 0644)
100-
remoteWt.Add("ignore.json")
101-
remoteWt.Add("CVE-A.json")
88+
if err := os.WriteFile(filepath.Join(remoteDir, "ignore.json"), []byte("{}"), 0600); err != nil {
89+
t.Fatalf("Failed to write file: %v", err)
90+
}
91+
if err := os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte("{}"), 0600); err != nil {
92+
t.Fatalf("Failed to write file: %v", err)
93+
}
94+
if _, err := remoteWt.Add("ignore.json"); err != nil {
95+
t.Fatalf("Failed to add file: %v", err)
96+
}
97+
if _, err := remoteWt.Add("CVE-A.json"); err != nil {
98+
t.Fatalf("Failed to add file: %v", err)
99+
}
102100
commitA, _ := remoteWt.Commit("Initial", &git.CommitOptions{
103101
Author: &object.Signature{Name: "Test", Email: "test@example.com", When: time.Now()},
104102
})
105103

106104
// Second commit: Modify old file, add new file
107-
os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte(`{"modified": true}`), 0644)
108-
os.WriteFile(filepath.Join(remoteDir, "CVE-B.json"), []byte("{}"), 0644)
109-
remoteWt.Add("CVE-A.json")
110-
remoteWt.Add("CVE-B.json")
105+
if err := os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte(`{"modified": true}`), 0600); err != nil {
106+
t.Fatalf("Failed to write file: %v", err)
107+
}
108+
if err := os.WriteFile(filepath.Join(remoteDir, "CVE-B.json"), []byte("{}"), 0600); err != nil {
109+
t.Fatalf("Failed to write file: %v", err)
110+
}
111+
if _, err := remoteWt.Add("CVE-A.json"); err != nil {
112+
t.Fatalf("Failed to add file: %v", err)
113+
}
114+
if _, err := remoteWt.Add("CVE-B.json"); err != nil {
115+
t.Fatalf("Failed to add file: %v", err)
116+
}
111117
commitB, _ := remoteWt.Commit("Second", &git.CommitOptions{
112118
Author: &object.Signature{Name: "Test", Email: "test@example.com", When: time.Now()},
113119
})
114120

115121
mockStore := &mockSourceRepositoryStore{
116-
updates: make(map[string]interface{}),
122+
updates: make(map[string]any),
117123
}
118-
workDir, _ := os.MkdirTemp("", "osv-git-work-*")
119-
defer os.RemoveAll(workDir)
124+
workDir := t.TempDir()
120125

121126
config := Config{
122127
SourceRepoStore: mockStore,
@@ -141,7 +146,7 @@ func TestHandleImportGit(t *testing.T) {
141146
}
142147
close(ch)
143148

144-
var records []gitSourceRecord
149+
records := make([]gitSourceRecord, 0, 10)
145150
for r := range ch {
146151
records = append(records, r.(gitSourceRecord))
147152
}
@@ -172,11 +177,7 @@ func TestHandleImportGit(t *testing.T) {
172177

173178
func TestHandleImportGit_Deletion(t *testing.T) {
174179
// Setup a temporary git repo acting as the remote source
175-
remoteDir, err := os.MkdirTemp("", "osv-git-remote-del-*")
176-
if err != nil {
177-
t.Fatalf("Failed to create temp dir: %v", err)
178-
}
179-
defer os.RemoveAll(remoteDir)
180+
remoteDir := t.TempDir()
180181

181182
remoteRepo, err := git.PlainInit(remoteDir, false)
182183
if err != nil {
@@ -188,8 +189,12 @@ func TestHandleImportGit_Deletion(t *testing.T) {
188189
}
189190

190191
// Initial commit: one file
191-
os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte("{}"), 0644)
192-
remoteWt.Add("CVE-A.json")
192+
if err := os.WriteFile(filepath.Join(remoteDir, "CVE-A.json"), []byte("{}"), 0600); err != nil {
193+
t.Fatalf("Failed to write file: %v", err)
194+
}
195+
if _, err := remoteWt.Add("CVE-A.json"); err != nil {
196+
t.Fatalf("Failed to add file: %v", err)
197+
}
193198
commitA, _ := remoteWt.Commit("Initial", &git.CommitOptions{
194199
Author: &object.Signature{Name: "Test", Email: "test@example.com", When: time.Now()},
195200
})
@@ -201,10 +206,9 @@ func TestHandleImportGit_Deletion(t *testing.T) {
201206
})
202207

203208
mockStore := &mockSourceRepositoryStore{
204-
updates: make(map[string]interface{}),
209+
updates: make(map[string]any),
205210
}
206-
workDir, _ := os.MkdirTemp("", "osv-git-work-del-*")
207-
defer os.RemoveAll(workDir)
211+
workDir := t.TempDir()
208212

209213
config := Config{
210214
SourceRepoStore: mockStore,
@@ -228,7 +232,7 @@ func TestHandleImportGit_Deletion(t *testing.T) {
228232
}
229233
close(ch)
230234

231-
var records []gitSourceRecord
235+
records := make([]gitSourceRecord, 0, 10)
232236
for r := range ch {
233237
records = append(records, r.(gitSourceRecord))
234238
}

0 commit comments

Comments
 (0)