Skip to content

Add last commit size for repository #1243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HaiHui886 merged 1 commit into from
Jul 9, 2026
Merged

Add last commit size for repository #1243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions _mocks/opencsg.com/csghub-server/builder/git/gitserver/mock_GitServer.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions _mocks/opencsg.com/csghub-server/builder/rpc/mock_UserSvcClient.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions _mocks/opencsg.com/csghub-server/builder/store/database/mock_UserStore.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions _mocks/opencsg.com/csghub-server/component/mock_RepoComponent.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions api/handler/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2027,7 +2027,7 @@ func (h *RepoHandler) GetRepoSizeByBranch(ctx *gin.Context) {
branch := ctx.Param("branch")
repoType := common.RepoTypeFromContext(ctx)

size, err := h.c.GetRepoSizeByBranch(ctx.Request.Context(), repoType, namespace, name, branch, currentUser)
resp, err := h.c.GetRepoSizeByBranch(ctx.Request.Context(), repoType, namespace, name, branch, currentUser)
if err != nil {
slog.ErrorContext(ctx.Request.Context(), "Failed to get repo size", slog.String("repo_type", string(repoType)), slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch))
if errors.Is(err, errorx.ErrForbidden) {
Expand All @@ -2042,8 +2042,8 @@ func (h *RepoHandler) GetRepoSizeByBranch(ctx *gin.Context) {
return
}

slog.Debug("Get repo size succeed", slog.String("repo_type", string(repoType)), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch), slog.Any("size", size))
httpbase.OK(ctx, size)
slog.Debug("Get repo size succeed", slog.String("repo_type", string(repoType)), slog.String("namespace", namespace), slog.String("name", name), slog.String("branch", branch), slog.Any("total_size", resp.TotalSize), slog.Any("last_commit_size", resp.LastCommitSize))
httpbase.OK(ctx, resp)
}

// BatchGetRepoExtra godoc
Expand Down
17 changes: 10 additions & 7 deletions api/handler/repo_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,14 @@ func TestRepoHandler_GetRepoSizeByBranch(t *testing.T) {
tester.WithKV("repo_type", types.ModelRepo)

// Set mock expectation
expectedSize := int64(1024)
tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(expectedSize, nil)
expectedResp := types.RepoSizeResponse{TotalSize: 1024, LastCommitSize: 512}
tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(expectedResp, nil)

// Execute request
tester.Execute()

// Verify response
tester.ResponseEq(t, http.StatusOK, tester.OKText, expectedSize)
tester.ResponseEq(t, http.StatusOK, tester.OKText, expectedResp)
}

func TestRepoHandler_ScanIndustryTags(t *testing.T) {
Expand Down Expand Up @@ -186,7 +186,7 @@ func TestRepoHandler_GetRepoSizeByBranch_Error(t *testing.T) {
tester.WithKV("repo_type", types.ModelRepo)

// Set mock expectation with error
tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(int64(0), errors.New("failed to get repo size"))
tester.mocks.repo.EXPECT().GetRepoSizeByBranch(tester.Ctx(), types.ModelRepo, "u", "r", "main", "u").Return(types.RepoSizeResponse{}, errors.New("failed to get repo size"))

// Execute request
tester.Execute()
Expand Down Expand Up @@ -1985,13 +1985,16 @@ func TestRepoHandler_BatchGetRepoExtra(t *testing.T) {
})
tester.WithBody(t, types.BatchRepoExtraReq{RepoIDs: []int64{1, 2}}).WithUser()

tester.mocks.repo.EXPECT().BatchGetRepoExtra(tester.Ctx(), []int64{1, 2}, "u").Return([]types.RepoExtraItem{{RepoID: 1, Size: 100}, {RepoID: 2, Size: 200}}, nil)
tester.mocks.repo.EXPECT().BatchGetRepoExtra(tester.Ctx(), []int64{1, 2}, "u").Return([]types.RepoExtraItem{
{RepoID: 1, Size: 100, LastCommitSize: 50},
{RepoID: 2, Size: 200, LastCommitSize: 80},
}, nil)

tester.Execute()

tester.ResponseEq(t, http.StatusOK, tester.OKText, []types.RepoExtraItem{
{RepoID: 1, Size: 100},
{RepoID: 2, Size: 200},
{RepoID: 1, Size: 100, LastCommitSize: 50},
{RepoID: 2, Size: 200, LastCommitSize: 80},
})
})

Expand Down
133 changes: 133 additions & 0 deletions builder/git/gitserver/gitaly/repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,139 @@ func (c *Client) GetRepoLfsSize(ctx context.Context, req gitserver.GetRepoInfoBy
return totalSize, nil
}

const getBlobsBatchSize = 1000

func (c *Client) GetLastCommitSize(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()

errCtx := errorx.Ctx().
Set("repo_type", req.RepoType).
Set("namespace", req.Namespace).
Set("name", req.Name).
Set("ref", req.Ref)

relativePath, err := c.BuildRelativePath(ctx, req.RepoType, req.Namespace, req.Name)
if err != nil {
return 0, err
}

repository := &gitalypb.Repository{
StorageName: c.config.GitalyServer.Storage,
RelativePath: relativePath,
}

// Get all file paths at HEAD
listFilesReq := &gitalypb.ListFilesRequest{
Repository: repository,
Revision: []byte(req.Ref),
}
filesStream, err := c.commitClient.ListFiles(ctx, listFilesReq)
if err != nil {
return 0, errorx.ErrGitListFilesFailed(err, errCtx)
}

var allPaths []string
for {
filesResp, err := filesStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return 0, errorx.ErrGitListFilesFailed(err, errCtx)
}
if filesResp != nil {
for _, path := range filesResp.Paths {
allPaths = append(allPaths, string(path))
}
}
}

if len(allPaths) == 0 {
return 0, nil
}

// Get blob sizes in batches to avoid exceeding gRPC message limits
var (
lastCommitSize int64
blobIDs []string
)
for i := 0; i < len(allPaths); i += getBlobsBatchSize {
end := i + getBlobsBatchSize
if end > len(allPaths) {
end = len(allPaths)
}
batch := allPaths[i:end]

var revisionPaths []*gitalypb.GetBlobsRequest_RevisionPath
for _, path := range batch {
revisionPaths = append(revisionPaths, &gitalypb.GetBlobsRequest_RevisionPath{
Revision: req.Ref,
Path: []byte(path),
})
}

blobsStream, err := c.blobClient.GetBlobs(ctx, &gitalypb.GetBlobsRequest{
Repository: repository,
RevisionPaths: revisionPaths,
Limit: 0,
})
if err != nil {
return 0, errorx.ErrGitGetBlobsFailed(err, errCtx)
}

for {
blobResp, err := blobsStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return 0, errorx.ErrGitGetBlobsFailed(err, errCtx)
}
if blobResp != nil {
lastCommitSize += blobResp.Size
if blobResp.Oid != "" {
blobIDs = append(blobIDs, blobResp.Oid)
}
}
}
}

// Get LFS file sizes in batches
for i := 0; i < len(blobIDs); i += getBlobsBatchSize {
end := i + getBlobsBatchSize
if end > len(blobIDs) {
end = len(blobIDs)
}
batch := blobIDs[i:end]

pointersStream, err := c.blobClient.GetLFSPointers(ctx, &gitalypb.GetLFSPointersRequest{
Repository: repository,
BlobIds: batch,
})
if err != nil {
return 0, errorx.ErrGitGetLfsPointersFailed(err, errCtx)
}

for {
pointerResp, err := pointersStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return 0, errorx.ErrGitGetLfsPointersFailed(err, errCtx)
}
if pointerResp != nil {
for _, pointer := range pointerResp.LfsPointers {
lastCommitSize += pointer.FileSize
}
}
}
}

return lastCommitSize, nil
}

func (c *Client) CreateFork(ctx context.Context, req gitserver.CreateForkReq) error {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
Expand Down
Loading
Loading