diff --git a/_mocks/opencsg.com/csghub-server/builder/git/gitserver/mock_GitServer.go b/_mocks/opencsg.com/csghub-server/builder/git/gitserver/mock_GitServer.go index 9124fea48..332797224 100644 --- a/_mocks/opencsg.com/csghub-server/builder/git/gitserver/mock_GitServer.go +++ b/_mocks/opencsg.com/csghub-server/builder/git/gitserver/mock_GitServer.go @@ -1278,6 +1278,63 @@ func (_c *MockGitServer_GetFilesByRevisionAndPaths_Call) RunAndReturn(run func(c return _c } +// GetLastCommitSize provides a mock function with given fields: ctx, req +func (_m *MockGitServer) GetLastCommitSize(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (int64, error) { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for GetLastCommitSize") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, gitserver.GetRepoInfoByPathReq) (int64, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, gitserver.GetRepoInfoByPathReq) int64); ok { + r0 = rf(ctx, req) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(context.Context, gitserver.GetRepoInfoByPathReq) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockGitServer_GetLastCommitSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLastCommitSize' +type MockGitServer_GetLastCommitSize_Call struct { + *mock.Call +} + +// GetLastCommitSize is a helper method to define mock.On call +// - ctx context.Context +// - req gitserver.GetRepoInfoByPathReq +func (_e *MockGitServer_Expecter) GetLastCommitSize(ctx interface{}, req interface{}) *MockGitServer_GetLastCommitSize_Call { + return &MockGitServer_GetLastCommitSize_Call{Call: _e.mock.On("GetLastCommitSize", ctx, req)} +} + +func (_c *MockGitServer_GetLastCommitSize_Call) Run(run func(ctx context.Context, req gitserver.GetRepoInfoByPathReq)) *MockGitServer_GetLastCommitSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(gitserver.GetRepoInfoByPathReq)) + }) + return _c +} + +func (_c *MockGitServer_GetLastCommitSize_Call) Return(_a0 int64, _a1 error) *MockGitServer_GetLastCommitSize_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockGitServer_GetLastCommitSize_Call) RunAndReturn(run func(context.Context, gitserver.GetRepoInfoByPathReq) (int64, error)) *MockGitServer_GetLastCommitSize_Call { + _c.Call.Return(run) + return _c +} + // GetLogsTree provides a mock function with given fields: ctx, req func (_m *MockGitServer) GetLogsTree(ctx context.Context, req types.GetLogsTreeRequest) (*types.LogsTreeResp, error) { ret := _m.Called(ctx, req) diff --git a/_mocks/opencsg.com/csghub-server/builder/rpc/mock_UserSvcClient.go b/_mocks/opencsg.com/csghub-server/builder/rpc/mock_UserSvcClient.go index 9c1857404..fa02af9f6 100644 --- a/_mocks/opencsg.com/csghub-server/builder/rpc/mock_UserSvcClient.go +++ b/_mocks/opencsg.com/csghub-server/builder/rpc/mock_UserSvcClient.go @@ -5,10 +5,13 @@ package rpc import ( context "context" - mock "github.com/stretchr/testify/mock" membership "opencsg.com/csghub-server/builder/git/membership" - rpc "opencsg.com/csghub-server/builder/rpc" database "opencsg.com/csghub-server/builder/store/database" + + mock "github.com/stretchr/testify/mock" + + rpc "opencsg.com/csghub-server/builder/rpc" + types "opencsg.com/csghub-server/common/types" ) diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_UserStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_UserStore.go index da4fe6d26..9d951b201 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_UserStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_UserStore.go @@ -7,6 +7,7 @@ import ( mock "github.com/stretchr/testify/mock" database "opencsg.com/csghub-server/builder/store/database" + types "opencsg.com/csghub-server/common/types" ) diff --git a/_mocks/opencsg.com/csghub-server/component/mock_RepoComponent.go b/_mocks/opencsg.com/csghub-server/component/mock_RepoComponent.go index ec93808d4..3a6ad794d 100644 --- a/_mocks/opencsg.com/csghub-server/component/mock_RepoComponent.go +++ b/_mocks/opencsg.com/csghub-server/component/mock_RepoComponent.go @@ -2785,22 +2785,22 @@ func (_c *MockRepoComponent_GetNamespaceBillingUUID_Call) RunAndReturn(run func( } // GetRepoSizeByBranch provides a mock function with given fields: ctx, repoType, namespace, name, branch, currentUser -func (_m *MockRepoComponent) GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace string, name string, branch string, currentUser string) (int64, error) { +func (_m *MockRepoComponent) GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace string, name string, branch string, currentUser string) (types.RepoSizeResponse, error) { ret := _m.Called(ctx, repoType, namespace, name, branch, currentUser) if len(ret) == 0 { panic("no return value specified for GetRepoSizeByBranch") } - var r0 int64 + var r0 types.RepoSizeResponse var r1 error - if rf, ok := ret.Get(0).(func(context.Context, types.RepositoryType, string, string, string, string) (int64, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, types.RepositoryType, string, string, string, string) (types.RepoSizeResponse, error)); ok { return rf(ctx, repoType, namespace, name, branch, currentUser) } - if rf, ok := ret.Get(0).(func(context.Context, types.RepositoryType, string, string, string, string) int64); ok { + if rf, ok := ret.Get(0).(func(context.Context, types.RepositoryType, string, string, string, string) types.RepoSizeResponse); ok { r0 = rf(ctx, repoType, namespace, name, branch, currentUser) } else { - r0 = ret.Get(0).(int64) + r0 = ret.Get(0).(types.RepoSizeResponse) } if rf, ok := ret.Get(1).(func(context.Context, types.RepositoryType, string, string, string, string) error); ok { @@ -2835,12 +2835,12 @@ func (_c *MockRepoComponent_GetRepoSizeByBranch_Call) Run(run func(ctx context.C return _c } -func (_c *MockRepoComponent_GetRepoSizeByBranch_Call) Return(_a0 int64, _a1 error) *MockRepoComponent_GetRepoSizeByBranch_Call { +func (_c *MockRepoComponent_GetRepoSizeByBranch_Call) Return(_a0 types.RepoSizeResponse, _a1 error) *MockRepoComponent_GetRepoSizeByBranch_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockRepoComponent_GetRepoSizeByBranch_Call) RunAndReturn(run func(context.Context, types.RepositoryType, string, string, string, string) (int64, error)) *MockRepoComponent_GetRepoSizeByBranch_Call { +func (_c *MockRepoComponent_GetRepoSizeByBranch_Call) RunAndReturn(run func(context.Context, types.RepositoryType, string, string, string, string) (types.RepoSizeResponse, error)) *MockRepoComponent_GetRepoSizeByBranch_Call { _c.Call.Return(run) return _c } diff --git a/api/handler/repo.go b/api/handler/repo.go index 760daeb31..e38d13b40 100644 --- a/api/handler/repo.go +++ b/api/handler/repo.go @@ -2027,7 +2027,7 @@ func (h *RepoHandler) GetRepoSizeByBranch(ctx *gin.Context) { branch := ctx.Param("branch") repoType := common.RepoTypeFromContext(ctx) - size, err := h.c.GetRepoSizeByBranch(ctx.Request.Context(), repoType, namespace, name, branch, currentUser) + resp, err := h.c.GetRepoSizeByBranch(ctx.Request.Context(), repoType, namespace, name, branch, currentUser) if err != nil { slog.ErrorContext(ctx.Request.Context(), "Failed to get repo size", slog.String("repo_type", string(repoType)), slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch)) if errors.Is(err, errorx.ErrForbidden) { @@ -2042,8 +2042,8 @@ func (h *RepoHandler) GetRepoSizeByBranch(ctx *gin.Context) { return } - slog.Debug("Get repo size succeed", slog.String("repo_type", string(repoType)), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch), slog.Any("size", size)) - httpbase.OK(ctx, size) + slog.Debug("Get repo size succeed", slog.String("repo_type", string(repoType)), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch), slog.Any("total_size", resp.TotalSize), slog.Any("last_commit_size", resp.LastCommitSize)) + httpbase.OK(ctx, resp) } // BatchGetRepoExtra godoc diff --git a/api/handler/repo_test.go b/api/handler/repo_test.go index 6f573c654..c39aaddcd 100644 --- a/api/handler/repo_test.go +++ b/api/handler/repo_test.go @@ -82,14 +82,14 @@ func TestRepoHandler_GetRepoSizeByBranch(t *testing.T) { tester.WithKV("repo_type", types.ModelRepo) // Set mock expectation - expectedSize := int64(1024) - tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(expectedSize, nil) + expectedResp := types.RepoSizeResponse{TotalSize: 1024, LastCommitSize: 512} + tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(expectedResp, nil) // Execute request tester.Execute() // Verify response - tester.ResponseEq(t, http.StatusOK, tester.OKText, expectedSize) + tester.ResponseEq(t, http.StatusOK, tester.OKText, expectedResp) } func TestRepoHandler_ScanIndustryTags(t *testing.T) { @@ -186,7 +186,7 @@ func TestRepoHandler_GetRepoSizeByBranch_Error(t *testing.T) { tester.WithKV("repo_type", types.ModelRepo) // Set mock expectation with error - tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(int64(0), errors.New("failed to get repo size")) + tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(types.RepoSizeResponse{}, errors.New("failed to get repo size")) // Execute request tester.Execute() @@ -1985,13 +1985,16 @@ func TestRepoHandler_BatchGetRepoExtra(t *testing.T) { }) tester.WithBody(t, types.BatchRepoExtraReq{RepoIDs: []int64{1, 2}}).WithUser() - tester.mocks.repo.EXPECT().BatchGetRepoExtra(tester.Ctx(), []int64{1, 2}, "u").Return([]types.RepoExtraItem{{RepoID: 1, Size: 100}, {RepoID: 2, Size: 200}}, nil) + tester.mocks.repo.EXPECT().BatchGetRepoExtra(tester.Ctx(), []int64{1, 2}, "u").Return([]types.RepoExtraItem{ + {RepoID: 1, Size: 100, LastCommitSize: 50}, + {RepoID: 2, Size: 200, LastCommitSize: 80}, + }, nil) tester.Execute() tester.ResponseEq(t, http.StatusOK, tester.OKText, []types.RepoExtraItem{ - {RepoID: 1, Size: 100}, - {RepoID: 2, Size: 200}, + {RepoID: 1, Size: 100, LastCommitSize: 50}, + {RepoID: 2, Size: 200, LastCommitSize: 80}, }) }) diff --git a/builder/git/gitserver/gitaly/repo.go b/builder/git/gitserver/gitaly/repo.go index cdbf775e6..3657520b0 100644 --- a/builder/git/gitserver/gitaly/repo.go +++ b/builder/git/gitserver/gitaly/repo.go @@ -387,6 +387,139 @@ func (c *Client) GetRepoLfsSize(ctx context.Context, req gitserver.GetRepoInfoBy return totalSize, nil } +const getBlobsBatchSize = 1000 + +func (c *Client) GetLastCommitSize(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + errCtx := errorx.Ctx(). + Set("repo_type", req.RepoType). + Set("namespace", req.Namespace). + Set("name", req.Name). + Set("ref", req.Ref) + + relativePath, err := c.BuildRelativePath(ctx, req.RepoType, req.Namespace, req.Name) + if err != nil { + return 0, err + } + + repository := &gitalypb.Repository{ + StorageName: c.config.GitalyServer.Storage, + RelativePath: relativePath, + } + + // Get all file paths at HEAD + listFilesReq := &gitalypb.ListFilesRequest{ + Repository: repository, + Revision: []byte(req.Ref), + } + filesStream, err := c.commitClient.ListFiles(ctx, listFilesReq) + if err != nil { + return 0, errorx.ErrGitListFilesFailed(err, errCtx) + } + + var allPaths []string + for { + filesResp, err := filesStream.Recv() + if err != nil { + if err == io.EOF { + break + } + return 0, errorx.ErrGitListFilesFailed(err, errCtx) + } + if filesResp != nil { + for _, path := range filesResp.Paths { + allPaths = append(allPaths, string(path)) + } + } + } + + if len(allPaths) == 0 { + return 0, nil + } + + // Get blob sizes in batches to avoid exceeding gRPC message limits + var ( + lastCommitSize int64 + blobIDs []string + ) + for i := 0; i < len(allPaths); i += getBlobsBatchSize { + end := i + getBlobsBatchSize + if end > len(allPaths) { + end = len(allPaths) + } + batch := allPaths[i:end] + + var revisionPaths []*gitalypb.GetBlobsRequest_RevisionPath + for _, path := range batch { + revisionPaths = append(revisionPaths, &gitalypb.GetBlobsRequest_RevisionPath{ + Revision: req.Ref, + Path: []byte(path), + }) + } + + blobsStream, err := c.blobClient.GetBlobs(ctx, &gitalypb.GetBlobsRequest{ + Repository: repository, + RevisionPaths: revisionPaths, + Limit: 0, + }) + if err != nil { + return 0, errorx.ErrGitGetBlobsFailed(err, errCtx) + } + + for { + blobResp, err := blobsStream.Recv() + if err != nil { + if err == io.EOF { + break + } + return 0, errorx.ErrGitGetBlobsFailed(err, errCtx) + } + if blobResp != nil { + lastCommitSize += blobResp.Size + if blobResp.Oid != "" { + blobIDs = append(blobIDs, blobResp.Oid) + } + } + } + } + + // Get LFS file sizes in batches + for i := 0; i < len(blobIDs); i += getBlobsBatchSize { + end := i + getBlobsBatchSize + if end > len(blobIDs) { + end = len(blobIDs) + } + batch := blobIDs[i:end] + + pointersStream, err := c.blobClient.GetLFSPointers(ctx, &gitalypb.GetLFSPointersRequest{ + Repository: repository, + BlobIds: batch, + }) + if err != nil { + return 0, errorx.ErrGitGetLfsPointersFailed(err, errCtx) + } + + for { + pointerResp, err := pointersStream.Recv() + if err != nil { + if err == io.EOF { + break + } + return 0, errorx.ErrGitGetLfsPointersFailed(err, errCtx) + } + if pointerResp != nil { + for _, pointer := range pointerResp.LfsPointers { + lastCommitSize += pointer.FileSize + } + } + } + } + + return lastCommitSize, nil +} + func (c *Client) CreateFork(ctx context.Context, req gitserver.CreateForkReq) error { ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() diff --git a/builder/git/gitserver/gitaly/repo_test.go b/builder/git/gitserver/gitaly/repo_test.go index ed5f4aedb..b94b3884b 100644 --- a/builder/git/gitserver/gitaly/repo_test.go +++ b/builder/git/gitserver/gitaly/repo_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "context" + "fmt" "testing" "github.com/stretchr/testify/mock" @@ -247,3 +248,178 @@ func TestStripZipPrefix(t *testing.T) { require.Contains(t, err.Error(), "no files matched prefix") }) } + +func TestGitalyRepo_GetLastCommitSize(t *testing.T) { + repoInfoReq := gitserver.GetRepoInfoByPathReq{ + Namespace: "ns", + Name: "repo", + Ref: "main", + Path: "", + RepoType: types.ModelRepo, + File: false, + } + + t.Run("success", func(t *testing.T) { + tester := newGitalyTester(t) + ctx := context.TODO() + + tester.mocks.repoStore.EXPECT().FindByPath(mock.Anything, types.ModelRepo, "ns", "repo").Return(&database.Repository{ + ID: 1, + Hashed: false, + }, nil) + + // ListFiles returns all file paths at HEAD + tester.mocks.commitClient.EXPECT().ListFiles(mock.Anything, &gitalypb.ListFilesRequest{ + Repository: &gitalypb.Repository{ + StorageName: "st", + RelativePath: "models_ns/repo.git", + }, + Revision: []byte("main"), + }).Return(&MockGrpcStreamClient[*gitalypb.ListFilesResponse]{ + data: []*gitalypb.ListFilesResponse{ + {Paths: [][]byte{[]byte("README.md"), []byte("model.pth")}}, + }, + }, nil) + + // GetBlobs returns blob sizes for all files + tester.mocks.blobClient.EXPECT().GetBlobs(mock.Anything, &gitalypb.GetBlobsRequest{ + Repository: &gitalypb.Repository{ + StorageName: "st", + RelativePath: "models_ns/repo.git", + }, + RevisionPaths: []*gitalypb.GetBlobsRequest_RevisionPath{ + {Revision: "main", Path: []byte("README.md")}, + {Revision: "main", Path: []byte("model.pth")}, + }, + Limit: 0, + }).Return(&MockGrpcStreamClient[*gitalypb.GetBlobsResponse]{ + data: []*gitalypb.GetBlobsResponse{ + {Size: 1024, Oid: "oid1", Path: []byte("README.md"), Type: gitalypb.ObjectType_BLOB}, + {Size: 2048, Oid: "oid2", Path: []byte("model.pth"), Type: gitalypb.ObjectType_BLOB}, + }, + }, nil) + + // GetLFSPointers returns LFS file sizes + tester.mocks.blobClient.EXPECT().GetLFSPointers(mock.Anything, &gitalypb.GetLFSPointersRequest{ + Repository: &gitalypb.Repository{ + StorageName: "st", + RelativePath: "models_ns/repo.git", + }, + BlobIds: []string{"oid1", "oid2"}, + }).Return(&MockGrpcStreamClient[*gitalypb.GetLFSPointersResponse]{ + data: []*gitalypb.GetLFSPointersResponse{ + { + LfsPointers: []*gitalypb.LFSPointer{ + {Size: 130, FileSize: 1000000, Oid: "oid2"}, + }, + }, + }, + }, nil) + + size, err := tester.GetLastCommitSize(ctx, repoInfoReq) + require.NoError(t, err) + // blob: 1024 + 2048 = 3072, LFS: 1000000, total: 1003072 + require.Equal(t, int64(1003072), size) + }) + + t.Run("empty repo", func(t *testing.T) { + tester := newGitalyTester(t) + ctx := context.TODO() + + tester.mocks.repoStore.EXPECT().FindByPath(mock.Anything, types.ModelRepo, "ns", "repo").Return(&database.Repository{ + ID: 1, + Hashed: false, + }, nil) + + tester.mocks.commitClient.EXPECT().ListFiles(mock.Anything, mock.Anything).Return(&MockGrpcStreamClient[*gitalypb.ListFilesResponse]{ + data: []*gitalypb.ListFilesResponse{}, + }, nil) + + size, err := tester.GetLastCommitSize(ctx, repoInfoReq) + require.NoError(t, err) + require.Equal(t, int64(0), size) + }) + + t.Run("ListFiles error", func(t *testing.T) { + tester := newGitalyTester(t) + ctx := context.TODO() + + tester.mocks.repoStore.EXPECT().FindByPath(mock.Anything, types.ModelRepo, "ns", "repo").Return(&database.Repository{ + ID: 1, + Hashed: false, + }, nil) + + tester.mocks.commitClient.EXPECT().ListFiles(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("list files failed")) + + _, err := tester.GetLastCommitSize(ctx, repoInfoReq) + require.Error(t, err) + }) + + t.Run("batches GetBlobs calls for large file counts", func(t *testing.T) { + tester := newGitalyTester(t) + ctx := context.TODO() + + tester.mocks.repoStore.EXPECT().FindByPath(mock.Anything, types.ModelRepo, "ns", "repo").Return(&database.Repository{ + ID: 1, + Hashed: false, + }, nil) + + // Build 1001 paths to trigger 2 batches (batch size = 1000) + var allPaths [][]byte + for j := 0; j < 1001; j++ { + allPaths = append(allPaths, []byte(fmt.Sprintf("file_%d.txt", j))) + } + tester.mocks.commitClient.EXPECT().ListFiles(mock.Anything, mock.Anything).Return(&MockGrpcStreamClient[*gitalypb.ListFilesResponse]{ + data: []*gitalypb.ListFilesResponse{ + {Paths: allPaths}, + }, + }, nil) + + // First batch: 1000 files + tester.mocks.blobClient.EXPECT().GetBlobs(mock.Anything, mock.MatchedBy(func(req *gitalypb.GetBlobsRequest) bool { + return len(req.RevisionPaths) == 1000 + })).Return(&MockGrpcStreamClient[*gitalypb.GetBlobsResponse]{ + data: []*gitalypb.GetBlobsResponse{ + {Size: 1, Oid: "oid1"}, + }, + }, nil) + + // Second batch: 1 file + tester.mocks.blobClient.EXPECT().GetBlobs(mock.Anything, mock.MatchedBy(func(req *gitalypb.GetBlobsRequest) bool { + return len(req.RevisionPaths) == 1 + })).Return(&MockGrpcStreamClient[*gitalypb.GetBlobsResponse]{ + data: []*gitalypb.GetBlobsResponse{ + {Size: 1, Oid: "oid2"}, + }, + }, nil) + + tester.mocks.blobClient.EXPECT().GetLFSPointers(mock.Anything, mock.Anything).Return(&MockGrpcStreamClient[*gitalypb.GetLFSPointersResponse]{ + data: []*gitalypb.GetLFSPointersResponse{}, + }, nil) + + size, err := tester.GetLastCommitSize(ctx, repoInfoReq) + require.NoError(t, err) + require.Equal(t, int64(2), size) + }) + + t.Run("GetBlobs error", func(t *testing.T) { + tester := newGitalyTester(t) + ctx := context.TODO() + + tester.mocks.repoStore.EXPECT().FindByPath(mock.Anything, types.ModelRepo, "ns", "repo").Return(&database.Repository{ + ID: 1, + Hashed: false, + }, nil) + + tester.mocks.commitClient.EXPECT().ListFiles(mock.Anything, mock.Anything).Return(&MockGrpcStreamClient[*gitalypb.ListFilesResponse]{ + data: []*gitalypb.ListFilesResponse{ + {Paths: [][]byte{[]byte("README.md")}}, + }, + }, nil) + + tester.mocks.blobClient.EXPECT().GetBlobs(mock.Anything, mock.Anything).Return(nil, fmt.Errorf("get blobs failed")) + + _, err := tester.GetLastCommitSize(ctx, repoInfoReq) + require.Error(t, err) + }) +} diff --git a/builder/git/gitserver/gitea/repo.go b/builder/git/gitserver/gitea/repo.go index 188e54eed..285edf50c 100644 --- a/builder/git/gitserver/gitea/repo.go +++ b/builder/git/gitserver/gitea/repo.go @@ -129,6 +129,10 @@ func (c *Client) GetRepoLfsSize(ctx context.Context, req gitserver.GetRepoInfoBy return 0, nil } +func (c *Client) GetLastCommitSize(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (int64, error) { + return 0, nil +} + func (c *Client) CreateFork(ctx context.Context, req gitserver.CreateForkReq) error { return nil } diff --git a/builder/git/gitserver/interface.go b/builder/git/gitserver/interface.go index a91941ecb..607e74334 100644 --- a/builder/git/gitserver/interface.go +++ b/builder/git/gitserver/interface.go @@ -87,6 +87,8 @@ type GitServer interface { GetRepoSize(ctx context.Context, req GetRepoInfoByPathReq) (int64, error) // GetRepoLfsSize returns the total size of all LFS files in the repository GetRepoLfsSize(ctx context.Context, req GetRepoInfoByPathReq) (int64, error) + // GetLastCommitSize returns the total size of files changed in the last commit (HEAD) + GetLastCommitSize(ctx context.Context, req GetRepoInfoByPathReq) (int64, error) // CreateFork creates a fork of a repository CreateFork(ctx context.Context, req CreateForkReq) error } diff --git a/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.down.sql b/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.down.sql new file mode 100644 index 000000000..ba8f560ad --- /dev/null +++ b/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.down.sql @@ -0,0 +1 @@ +ALTER TABLE repository_statistics DROP COLUMN last_commit_size; diff --git a/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.up.sql b/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.up.sql new file mode 100644 index 000000000..d7c4fbb56 --- /dev/null +++ b/builder/store/database/migrations/20260703055125_add_last_commit_size_to_repository_statistics.up.sql @@ -0,0 +1 @@ +ALTER TABLE repository_statistics ADD COLUMN last_commit_size BIGINT NOT NULL DEFAULT 0; diff --git a/builder/store/database/repository_statistics.go b/builder/store/database/repository_statistics.go index a29858e1d..5c400f553 100644 --- a/builder/store/database/repository_statistics.go +++ b/builder/store/database/repository_statistics.go @@ -15,9 +15,10 @@ type RepositoryStatistics struct { Branch string `bun:"branch,notnull"` TotalSize int64 `bun:"total_size,notnull,default:0"` NonLfsSize int64 `bun:"non_lfs_size,notnull,default:0"` - LfsSize int64 `bun:"lfs_size,notnull,default:0"` - CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp"` + LfsSize int64 `bun:"lfs_size,notnull,default:0"` + LastCommitSize int64 `bun:"last_commit_size,notnull,default:0"` + CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp"` // Relations Repository *Repository `bun:"rel:belongs-to,join:repository_id=id"` diff --git a/common/types/repo.go b/common/types/repo.go index f0c1c0ee5..160dd3840 100644 --- a/common/types/repo.go +++ b/common/types/repo.go @@ -431,6 +431,12 @@ type BatchRepoExtraReq struct { } type RepoExtraItem struct { - RepoID int64 `json:"repo_id"` - Size int64 `json:"size"` + RepoID int64 `json:"repo_id"` + Size int64 `json:"size"` + LastCommitSize int64 `json:"last_commit_size"` +} + +type RepoSizeResponse struct { + TotalSize int64 `json:"total_size"` + LastCommitSize int64 `json:"last_commit_size"` } diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index 7efc651b2..eb2d14ad6 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -685,18 +685,26 @@ func (c *gitCallbackComponentImpl) CalculateRepoSize(ctx context.Context, req *t // Calculate total size totalSize := nonLfsSize + lfsSize + // Calculate last commit size + lastCommitSize, err := c.gitServer.GetLastCommitSize(ctx, repoInfoReq) + if err != nil { + slog.Error("failed to get last commit size", slog.Any("error", err), slog.Any("repo_id", repo.ID), slog.String("branch", branchName)) + return err + } + // Check if repository statistics already exists for this branch existingStats, err := c.repositoryStatisticsStore.FindByRepositoryIDAndBranch(ctx, repo.ID, branchName) if err != nil || existingStats == nil { // If not found, create a new record stats := &database.RepositoryStatistics{ - RepositoryID: repo.ID, - Branch: branchName, - TotalSize: totalSize, - NonLfsSize: nonLfsSize, - LfsSize: lfsSize, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + RepositoryID: repo.ID, + Branch: branchName, + TotalSize: totalSize, + NonLfsSize: nonLfsSize, + LfsSize: lfsSize, + LastCommitSize: lastCommitSize, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), } err = c.repositoryStatisticsStore.Create(ctx, stats) if err != nil { @@ -708,6 +716,7 @@ func (c *gitCallbackComponentImpl) CalculateRepoSize(ctx context.Context, req *t existingStats.TotalSize = totalSize existingStats.NonLfsSize = nonLfsSize existingStats.LfsSize = lfsSize + existingStats.LastCommitSize = lastCommitSize existingStats.UpdatedAt = time.Now() err = c.repositoryStatisticsStore.Update(ctx, existingStats) if err != nil { @@ -716,7 +725,7 @@ func (c *gitCallbackComponentImpl) CalculateRepoSize(ctx context.Context, req *t } } - slog.Info("calculated repo size for branch", slog.Any("repo_id", repo.ID), slog.String("branch", branchName), slog.Any("total_size", totalSize), slog.Any("non_lfs_size", nonLfsSize), slog.Any("lfs_size", lfsSize)) + slog.Info("calculated repo size for branch", slog.Any("repo_id", repo.ID), slog.String("branch", branchName), slog.Any("total_size", totalSize), slog.Any("non_lfs_size", nonLfsSize), slog.Any("lfs_size", lfsSize), slog.Any("last_commit_size", lastCommitSize)) return nil } diff --git a/component/callback/git_callback_test.go b/component/callback/git_callback_test.go index e7ea5ac0b..98826bb81 100644 --- a/component/callback/git_callback_test.go +++ b/component/callback/git_callback_test.go @@ -242,6 +242,7 @@ func TestGitCallbackComponentImpl_CalculateRepoSize(t *testing.T) { } gc.mocks.gitServer.EXPECT().GetRepoSize(ctx, repoInfoReq).Return(int64(1024), nil) gc.mocks.gitServer.EXPECT().GetRepoLfsSize(ctx, repoInfoReq).Return(int64(2048), nil) + gc.mocks.gitServer.EXPECT().GetLastCommitSize(ctx, repoInfoReq).Return(int64(512), nil) // Expectations for repository statistics gc.mocks.stores.RepositoryStatisticsMock().EXPECT().FindByRepositoryIDAndBranch(ctx, repo.ID, branchName).Return(nil, nil) @@ -274,14 +275,16 @@ func TestGitCallbackComponentImpl_CalculateRepoSize(t *testing.T) { } gc.mocks.gitServer.EXPECT().GetRepoSize(ctx, repoInfoReq).Return(int64(1024), nil) gc.mocks.gitServer.EXPECT().GetRepoLfsSize(ctx, repoInfoReq).Return(int64(2048), nil) + gc.mocks.gitServer.EXPECT().GetLastCommitSize(ctx, repoInfoReq).Return(int64(512), nil) // Expectations for repository statistics existingStats := &database.RepositoryStatistics{ - RepositoryID: repo.ID, - Branch: branchName, - TotalSize: 1000, - NonLfsSize: 500, - LfsSize: 500, + RepositoryID: repo.ID, + Branch: branchName, + TotalSize: 1000, + NonLfsSize: 500, + LfsSize: 500, + LastCommitSize: 256, } gc.mocks.stores.RepositoryStatisticsMock().EXPECT().FindByRepositoryIDAndBranch(ctx, repo.ID, branchName).Return(existingStats, nil) gc.mocks.stores.RepositoryStatisticsMock().EXPECT().Update(ctx, mock.Anything).Return(nil) diff --git a/component/repo.go b/component/repo.go index 4ae0a35f2..fa4d52315 100644 --- a/component/repo.go +++ b/component/repo.go @@ -197,7 +197,7 @@ type RepoComponent interface { DeletePendingDeletion(ctx context.Context) error GetRepos(ctx context.Context, search, currentUser string, repoType types.RepositoryType) ([]string, error) // GetRepoSizeByBranch gets the repository size for a specific branch - GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace, name, branch, currentUser string) (int64, error) + GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace, name, branch, currentUser string) (types.RepoSizeResponse, error) // BatchGetRepoExtra gets the default branch size for multiple repositories BatchGetRepoExtra(ctx context.Context, repoIDs []int64, currentUser string) ([]types.RepoExtraItem, error) // DownloadCodeZip downloads the code repository as a zip archive @@ -3282,30 +3282,33 @@ func (c *repoComponentImpl) GetRepos(ctx context.Context, search, currentUser st } // GetRepoSizeByBranch gets the repository size for a specific branch -func (c *repoComponentImpl) GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace, name, branch, currentUser string) (int64, error) { +func (c *repoComponentImpl) GetRepoSizeByBranch(ctx context.Context, repoType types.RepositoryType, namespace, name, branch, currentUser string) (types.RepoSizeResponse, error) { repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name) if err != nil { - return 0, fmt.Errorf("failed to find repo, error: %w", err) + return types.RepoSizeResponse{}, fmt.Errorf("failed to find repo, error: %w", err) } permission, err := c.GetUserRepoPermission(ctx, currentUser, repo) if err != nil { - return 0, fmt.Errorf("failed to get user repo permission, error: %w", err) + return types.RepoSizeResponse{}, fmt.Errorf("failed to get user repo permission, error: %w", err) } if !permission.CanRead { - return 0, errorx.ErrForbiddenMsg("users do not have permission to get repo size in this repo") + return types.RepoSizeResponse{}, errorx.ErrForbiddenMsg("users do not have permission to get repo size in this repo") } // Get repository statistics from database stats, err := c.repoStatisticsStore.FindByRepositoryIDAndBranch(ctx, repo.ID, branch) if err != nil { if errors.Is(err, errorx.ErrNotFound) { - return 0, errorx.ErrNotFound + return types.RepoSizeResponse{}, errorx.ErrNotFound } - return 0, fmt.Errorf("failed to get repo statistics, error: %w", err) + return types.RepoSizeResponse{}, fmt.Errorf("failed to get repo statistics, error: %w", err) } - return stats.TotalSize, nil + return types.RepoSizeResponse{ + TotalSize: stats.TotalSize, + LastCommitSize: stats.LastCommitSize, + }, nil } // BatchGetRepoExtra gets the default branch size for multiple repositories by their IDs. @@ -3353,6 +3356,7 @@ func (c *repoComponentImpl) BatchGetRepoExtra(ctx context.Context, repoIDs []int } sizeMap := make(map[int64]int64, len(readableRepos)) + lastCommitSizeMap := make(map[int64]int64, len(readableRepos)) if len(repoDefaultBranches) > 0 { repoIDsWithBranch := make([]int64, 0, len(repoDefaultBranches)) @@ -3369,13 +3373,18 @@ func (c *repoComponentImpl) BatchGetRepoExtra(ctx context.Context, repoIDs []int for _, stat := range stats { if defaultBranch, ok := repoDefaultBranches[stat.RepositoryID]; ok && stat.Branch == defaultBranch { sizeMap[stat.RepositoryID] = stat.TotalSize + lastCommitSizeMap[stat.RepositoryID] = stat.LastCommitSize } } } result := make([]types.RepoExtraItem, 0, len(readableRepos)) for _, repo := range readableRepos { - result = append(result, types.RepoExtraItem{RepoID: repo.ID, Size: sizeMap[repo.ID]}) + result = append(result, types.RepoExtraItem{ + RepoID: repo.ID, + Size: sizeMap[repo.ID], + LastCommitSize: lastCommitSizeMap[repo.ID], + }) } slices.SortFunc(result, func(a, b types.RepoExtraItem) int { return cmp.Compare(a.RepoID, b.RepoID) diff --git a/component/repo_test.go b/component/repo_test.go index a3b7bbc27..2cb593c1d 100644 --- a/component/repo_test.go +++ b/component/repo_test.go @@ -2592,12 +2592,12 @@ func TestRepoComponent_GetRepoSizeByBranch(t *testing.T) { } repoComp.mocks.stores.RepoMock().EXPECT().FindByPath(mock.Anything, types.ModelRepo, ns.Path, repo.Name).Return(repo, nil) - stats := database.RepositoryStatistics{TotalSize: 1024} + stats := database.RepositoryStatistics{TotalSize: 1024, LastCommitSize: 512} repoComp.mocks.stores.RepositoryStatisticsMock().EXPECT().FindByRepositoryIDAndBranch(mock.Anything, repo.ID, "main").Return(&stats, nil) - size, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) + resp, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) require.Nil(t, err) - require.Equal(t, int64(1024), size) + require.Equal(t, types.RepoSizeResponse{TotalSize: 1024, LastCommitSize: 512}, resp) }) t.Run("should return error when repo not found", func(t *testing.T) { @@ -2606,9 +2606,9 @@ func TestRepoComponent_GetRepoSizeByBranch(t *testing.T) { repoComp.mocks.stores.RepoMock().EXPECT().FindByPath(mock.Anything, types.ModelRepo, "namespace", "name").Return(nil, errors.New("repo not found")) - size, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, "namespace", "name", "main", "user_name") + resp, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, "namespace", "name", "main", "user_name") require.Error(t, err) - require.Equal(t, int64(0), size) + require.Equal(t, types.RepoSizeResponse{}, resp) }) t.Run("should return error when user has no read permission", func(t *testing.T) { @@ -2632,9 +2632,9 @@ func TestRepoComponent_GetRepoSizeByBranch(t *testing.T) { } repoComp.mocks.stores.RepoMock().EXPECT().FindByPath(mock.Anything, types.ModelRepo, ns.Path, repo.Name).Return(repo, nil) - size, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) + resp, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) require.Error(t, err) - require.Equal(t, int64(0), size) + require.Equal(t, types.RepoSizeResponse{}, resp) }) t.Run("should return error when repo statistics not found", func(t *testing.T) { @@ -2660,9 +2660,9 @@ func TestRepoComponent_GetRepoSizeByBranch(t *testing.T) { repoComp.mocks.stores.RepositoryStatisticsMock().EXPECT().FindByRepositoryIDAndBranch(mock.Anything, repo.ID, "main").Return(nil, errorx.ErrNotFound) - size, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) + resp, err := repoComp.GetRepoSizeByBranch(ctx, types.ModelRepo, ns.Path, repo.Name, "main", user.Username) require.Error(t, err) - require.Equal(t, int64(0), size) + require.Equal(t, types.RepoSizeResponse{}, resp) require.True(t, errors.Is(err, errorx.ErrNotFound)) }) } @@ -2685,14 +2685,14 @@ func TestRepoComponent_BatchGetRepoExtra(t *testing.T) { repoComp.mocks.stores.RepoMock().EXPECT().FindByIds(ctx, []int64{1, 2}).Return(repos, nil) stats := []*database.RepositoryStatistics{ - {RepositoryID: 1, Branch: "main", TotalSize: 1024}, + {RepositoryID: 1, Branch: "main", TotalSize: 1024, LastCommitSize: 100}, {RepositoryID: 2, Branch: "main", TotalSize: 2048}, } repoComp.mocks.stores.RepositoryStatisticsMock().EXPECT().FindByRepositoryIDs(ctx, []int64{1, 2}).Return(stats, nil) result, err := repoComp.BatchGetRepoExtra(ctx, []int64{1, 2}, "admin") require.Nil(t, err) - require.Equal(t, []types.RepoExtraItem{{RepoID: 1, Size: 1024}, {RepoID: 2, Size: 2048}}, result) + require.Equal(t, []types.RepoExtraItem{{RepoID: 1, Size: 1024, LastCommitSize: 100}, {RepoID: 2, Size: 2048}}, result) }) t.Run("regular user can read own repos and public repos", func(t *testing.T) {