Skip to content

Commit b50144b

Browse files
bwalshmatthewpeterkort
bwalsh
and
matthewpeterkort
authored
fix/incomplete-add-url-3 (#197)
* feature/pre-commit-changes * deprecate/go-best-effort-is-lfs-tracked #196 * support buckets w/ prefix * latest data-client refactor/progress-callback * bump go-git dependabot * fix/lfs.customtransfer.drs.* #198 * fix:git-parsing #201 * Fix/url 3 pr issues (#199) * remove unused commands * fix:git-parsing #201 * filter out integration test * merge url-3-pr-issues * nil logger check * merged test results * fix:gitactions Getwd no such file/dir Co-authored-by: Brian Walsh <brian@bwalsh.com> Co-authored-by: Matthew Peterkort <33436238+matthewpeterkort@users.noreply.github.com>
1 parent feb6b09 commit b50144b

95 files changed

Lines changed: 45465 additions & 31368 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr-checks.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,4 @@ jobs:
7171
go-version-file: go.mod
7272

7373
- name: Run tests
74-
run: go test -v -race $(go list ./... | grep -v 'tests/integration' | grep -v 'client/indexd/tests')
74+
run: go test -v -race $( go list ./... | grep -v tests/integration )

‎client/anvil/anvil_client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import (
1010
"time"
1111

1212
"github.com/bytedance/sonic"
13-
drs "github.com/calypr/data-client/indexd/drs"
14-
hash "github.com/calypr/data-client/indexd/hash"
13+
drs "github.com/calypr/data-client/drs"
14+
hash "github.com/calypr/data-client/hash"
1515
"golang.org/x/oauth2/google"
1616
)
1717

‎client/indexd/add_url.go‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,17 @@ import (
1212
awsConfig "github.com/aws/aws-sdk-go-v2/config"
1313
"github.com/aws/aws-sdk-go-v2/credentials"
1414
"github.com/aws/aws-sdk-go-v2/service/s3"
15+
"github.com/calypr/data-client/drs"
1516
"github.com/calypr/data-client/fence"
17+
"github.com/calypr/data-client/hash"
1618
"github.com/calypr/data-client/indexd"
17-
"github.com/calypr/data-client/indexd/drs"
18-
"github.com/calypr/data-client/indexd/hash"
19+
"github.com/calypr/data-client/s3utils"
20+
"github.com/calypr/git-drs/cloud"
1921
"github.com/calypr/git-drs/common"
2022
"github.com/calypr/git-drs/drslog"
2123
"github.com/calypr/git-drs/drsmap"
2224
"github.com/calypr/git-drs/lfs"
2325
"github.com/calypr/git-drs/messages"
24-
"github.com/calypr/git-drs/s3_utils"
25-
"github.com/calypr/git-drs/utils"
2626
)
2727

2828
// getBucketDetails fetches bucket details from Gen3 using data-client.
@@ -32,7 +32,7 @@ func (inc *GitDrsIdxdClient) getBucketDetails(ctx context.Context, bucket string
3232

3333
// FetchS3MetadataWithBucketDetails fetches S3 metadata given bucket details.
3434
func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, bucketDetails *fence.S3Bucket, s3Client *s3.Client, logger *slog.Logger) (int64, string, error) {
35-
bucket, key, err := utils.ParseS3URL(s3URL)
35+
bucket, key, err := cloud.ParseS3URL(s3URL)
3636
if err != nil {
3737
return 0, "", fmt.Errorf("failed to parse S3 URL: %w", err)
3838
}
@@ -101,7 +101,7 @@ func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey,
101101
}
102102

103103
func (inc *GitDrsIdxdClient) fetchS3Metadata(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, s3Client *s3.Client, httpClient *http.Client, logger *slog.Logger) (int64, string, error) {
104-
bucket, _, err := utils.ParseS3URL(s3URL)
104+
bucket, _, err := cloud.ParseS3URL(s3URL)
105105
if err != nil {
106106
return 0, "", fmt.Errorf("failed to parse S3 URL: %w", err)
107107
}
@@ -145,7 +145,7 @@ func (inc *GitDrsIdxdClient) upsertIndexdRecord(ctx context.Context, url string,
145145

146146
// If no record exists, create one
147147
logger.Debug("creating new record")
148-
_, relPath, _ := utils.ParseS3URL(url)
148+
_, relPath, _ := cloud.ParseS3URL(url)
149149

150150
drsObj, err := drs.BuildDrsObj(relPath, sha256, fileSize, uuid, inc.Config.BucketName, projectId)
151151
if err != nil {
@@ -157,11 +157,11 @@ func (inc *GitDrsIdxdClient) upsertIndexdRecord(ctx context.Context, url string,
157157
return inc.RegisterRecord(ctx, drsObj)
158158
}
159159

160-
func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...s3_utils.AddURLOption) (s3_utils.S3Meta, error) {
160+
func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...cloud.AddURLOption) (s3utils.S3Meta, error) {
161161
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
162162
defer cancel()
163163

164-
cfg := &s3_utils.AddURLConfig{}
164+
cfg := &cloud.AddURLConfig{}
165165
for _, opt := range opts {
166166
opt(cfg)
167167
}
@@ -170,48 +170,48 @@ func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, r
170170
inc.Logger = drslog.NewNoOpLogger()
171171
}
172172

173-
if err := s3_utils.ValidateInputs(s3URL, sha256); err != nil {
174-
return s3_utils.S3Meta{}, err
173+
if err := s3utils.ValidateInputs(s3URL, sha256); err != nil {
174+
return s3utils.S3Meta{}, err
175175
}
176176

177-
_, relPath, err := utils.ParseS3URL(s3URL)
177+
_, relPath, err := cloud.ParseS3URL(s3URL)
178178
if err != nil {
179-
return s3_utils.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
179+
return s3utils.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
180180
}
181181

182-
isLFS, err := lfs.IsLFSTracked(".gitattributes", relPath)
182+
isLFS, err := lfs.IsLFSTracked(relPath)
183183
if err != nil {
184-
return s3_utils.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
184+
return s3utils.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
185185
}
186186
if !isLFS {
187-
return s3_utils.S3Meta{}, fmt.Errorf("file is not tracked by LFS")
187+
return s3utils.S3Meta{}, fmt.Errorf("file is not tracked by LFS")
188188
}
189189

190190
inc.Logger.Debug("Fetching S3 metadata...")
191191
fileSize, modifiedDate, err := inc.fetchS3Metadata(ctx, s3URL, awsAccessKey, awsSecretKey, regionFlag, endpointFlag, cfg.S3Client, cfg.HttpClient, inc.Logger)
192192
if err != nil {
193-
return s3_utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
193+
return s3utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
194194
}
195195

196196
inc.Logger.Debug(fmt.Sprintf("Fetched S3 metadata successfully: %d bytes, modified: %s", fileSize, modifiedDate))
197197

198198
inc.Logger.Debug("Processing indexd record...")
199199
drsObj, err := inc.upsertIndexdRecord(ctx, s3URL, sha256, fileSize, inc.Logger)
200200
if err != nil {
201-
return s3_utils.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
201+
return s3utils.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
202202
}
203203

204204
drsObjPath, err := drsmap.GetObjectPath(common.DRS_OBJS_PATH, drsObj.Checksums.SHA256)
205205
if err != nil {
206-
return s3_utils.S3Meta{}, err
206+
return s3utils.S3Meta{}, err
207207
}
208208
if err := drsmap.WriteDrsObj(drsObj, sha256, drsObjPath); err != nil {
209-
return s3_utils.S3Meta{}, err
209+
return s3utils.S3Meta{}, err
210210
}
211211

212212
inc.Logger.Debug("Indexd updated")
213213

214-
return s3_utils.S3Meta{
214+
return s3utils.S3Meta{
215215
Size: fileSize,
216216
LastModified: modifiedDate,
217217
}, nil

‎client/indexd/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ import (
88

99
"github.com/calypr/data-client/common"
1010
"github.com/calypr/data-client/conf"
11+
"github.com/calypr/data-client/drs"
1112
"github.com/calypr/data-client/g3client"
12-
"github.com/calypr/data-client/indexd/drs"
13-
"github.com/calypr/data-client/indexd/hash"
13+
"github.com/calypr/data-client/hash"
1414
"github.com/calypr/data-client/logs"
1515
"github.com/calypr/git-drs/client"
1616
"github.com/calypr/git-drs/drsmap"

‎client/indexd/client_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ import (
1515
"github.com/bytedance/sonic/encoder"
1616
"github.com/calypr/data-client/common"
1717
"github.com/calypr/data-client/conf"
18+
"github.com/calypr/data-client/drs"
1819
"github.com/calypr/data-client/g3client"
20+
"github.com/calypr/data-client/hash"
1921
"github.com/calypr/data-client/indexd"
20-
"github.com/calypr/data-client/indexd/drs"
21-
"github.com/calypr/data-client/indexd/hash"
2222
"github.com/calypr/data-client/logs"
2323
)
2424

‎client/indexd/register.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"strings"
88

99
"github.com/calypr/data-client/common"
10-
"github.com/calypr/data-client/indexd/drs"
10+
"github.com/calypr/data-client/drs"
1111
"github.com/calypr/data-client/upload"
1212
"github.com/calypr/git-drs/drsmap"
1313
)

‎client/interface.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ package client
33
import (
44
"context"
55

6+
drs "github.com/calypr/data-client/drs"
67
dataClient "github.com/calypr/data-client/g3client"
7-
drs "github.com/calypr/data-client/indexd/drs"
8-
hash "github.com/calypr/data-client/indexd/hash"
9-
"github.com/calypr/git-drs/s3_utils"
8+
hash "github.com/calypr/data-client/hash"
9+
"github.com/calypr/data-client/s3utils"
10+
"github.com/calypr/git-drs/cloud"
1011
)
1112

1213
type DRSClient interface {
@@ -59,7 +60,7 @@ type DRSClient interface {
5960
BuildDrsObj(fileName string, checksum string, size int64, drsId string) (*drs.DRSObject, error)
6061

6162
// Add an S3 URL to an existing indexd record
62-
AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...s3_utils.AddURLOption) (s3_utils.S3Meta, error)
63+
AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...cloud.AddURLOption) (s3utils.S3Meta, error)
6364

6465
GetBucketName() string
6566

‎client/tests/add-url-helper_test.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,8 @@ import (
66
"testing"
77

88
"github.com/bytedance/sonic/encoder"
9+
"github.com/calypr/git-drs/cloud"
910
"github.com/calypr/git-drs/drsmap"
10-
"github.com/calypr/git-drs/s3_utils"
11-
"github.com/calypr/git-drs/utils"
1211
)
1312

1413
// TestParseS3URL_Valid tests parsing valid S3 URLs
@@ -52,7 +51,7 @@ func TestParseS3URL_Valid(t *testing.T) {
5251

5352
for _, tt := range tests {
5453
t.Run(tt.name, func(t *testing.T) {
55-
bucket, key, err := utils.ParseS3URL(tt.s3URL)
54+
bucket, key, err := cloud.ParseS3URL(tt.s3URL)
5655
if (err != nil) != tt.wantErr {
5756
t.Errorf("ParseS3URL() error = %v, wantErr %v", err, tt.wantErr)
5857
return
@@ -106,7 +105,7 @@ func TestParseS3URL_Invalid(t *testing.T) {
106105

107106
for _, tt := range tests {
108107
t.Run(tt.name, func(t *testing.T) {
109-
_, _, err := utils.ParseS3URL(tt.s3URL)
108+
_, _, err := cloud.ParseS3URL(tt.s3URL)
110109
if (err != nil) != tt.wantErr {
111110
t.Errorf("ParseS3URL() error = %v, wantErr %v", err, tt.wantErr)
112111
}
@@ -149,8 +148,8 @@ func TestGetBucketDetails_Gen3Success(t *testing.T) {
149148
return
150149
}
151150

152-
response := s3_utils.S3BucketsResponse{
153-
S3Buckets: map[string]*s3_utils.S3Bucket{
151+
response := cloud.S3BucketsResponse{
152+
S3Buckets: map[string]*cloud.S3Bucket{
154153
"test-bucket": {
155154
Region: "us-west-2",
156155
EndpointURL: "https://s3.aws.amazon.com",

‎client/tests/add-url_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package indexd_tests
33
import (
44
"testing"
55

6-
"github.com/calypr/git-drs/s3_utils"
6+
"github.com/calypr/git-drs/cloud"
77
)
88

99
// TestValidateInputs_ValidInputs tests validation with valid S3 URL and SHA256
@@ -36,7 +36,7 @@ func TestValidateInputs_ValidInputs(t *testing.T) {
3636

3737
for _, tt := range tests {
3838
t.Run(tt.name, func(t *testing.T) {
39-
err := s3_utils.ValidateInputs(tt.s3URL, tt.sha256)
39+
err := cloud.ValidateInputs(tt.s3URL, tt.sha256)
4040
if (err != nil) != tt.wantErr {
4141
t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
4242
}
@@ -82,7 +82,7 @@ func TestValidateInputs_InvalidS3URL(t *testing.T) {
8282

8383
for _, tt := range tests {
8484
t.Run(tt.name, func(t *testing.T) {
85-
err := s3_utils.ValidateInputs(tt.s3URL, validSHA256)
85+
err := cloud.ValidateInputs(tt.s3URL, validSHA256)
8686
if (err != nil) != tt.wantErr {
8787
t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
8888
}
@@ -133,7 +133,7 @@ func TestValidateInputs_InvalidSHA256(t *testing.T) {
133133

134134
for _, tt := range tests {
135135
t.Run(tt.name, func(t *testing.T) {
136-
err := s3_utils.ValidateInputs(validS3URL, tt.sha256)
136+
err := cloud.ValidateInputs(validS3URL, tt.sha256)
137137
if (err != nil) != tt.wantErr {
138138
t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
139139
}
@@ -147,7 +147,7 @@ func TestValidateInputs_SHA256Normalization(t *testing.T) {
147147
uppercaseSHA256 := "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
148148

149149
// Should not error on uppercase SHA256 (it gets normalized internally)
150-
err := s3_utils.ValidateInputs(validS3URL, uppercaseSHA256)
150+
err := cloud.ValidateInputs(validS3URL, uppercaseSHA256)
151151
if err != nil {
152152
t.Errorf("validateInputs() should accept uppercase SHA256, got error: %v", err)
153153
}
@@ -159,15 +159,15 @@ func TestValidateInputs_HexDecodeValidation(t *testing.T) {
159159

160160
// Test valid 64-character hex string
161161
validHex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
162-
err := s3_utils.ValidateInputs(validS3URL, validHex)
162+
err := cloud.ValidateInputs(validS3URL, validHex)
163163
if err != nil {
164164
t.Errorf("validateInputs() error = %v, want nil", err)
165165
}
166166

167167
// Test that hex.DecodeString is properly checked
168168
// This has correct length but invalid hex
169169
invalidHex := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
170-
err = s3_utils.ValidateInputs(validS3URL, invalidHex)
170+
err = cloud.ValidateInputs(validS3URL, invalidHex)
171171
if err == nil {
172172
t.Errorf("validateInputs() should reject invalid hex, got nil error")
173173
}
@@ -201,7 +201,7 @@ func TestValidateInputs_CaseSensitivity(t *testing.T) {
201201

202202
for _, tt := range tests {
203203
t.Run(tt.name, func(t *testing.T) {
204-
err := s3_utils.ValidateInputs(tt.s3URL, validSHA256)
204+
err := cloud.ValidateInputs(tt.s3URL, validSHA256)
205205
if (err != nil) != tt.wantErr {
206206
t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
207207
}
@@ -245,7 +245,7 @@ func TestValidateInputs_EdgeCases(t *testing.T) {
245245

246246
for _, tt := range tests {
247247
t.Run(tt.name, func(t *testing.T) {
248-
err := s3_utils.ValidateInputs(tt.s3URL, tt.sha256)
248+
err := cloud.ValidateInputs(tt.s3URL, tt.sha256)
249249
if (err != nil) != tt.wantErr {
250250
t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
251251
}
@@ -260,6 +260,6 @@ func BenchmarkValidateInputs(b *testing.B) {
260260

261261
b.ResetTimer()
262262
for i := 0; i < b.N; i++ {
263-
_ = s3_utils.ValidateInputs(s3URL, sha256)
263+
_ = cloud.ValidateInputs(s3URL, sha256)
264264
}
265265
}

0 commit comments

Comments
 (0)