Skip to content

Commit 6a270b3

Browse files
XeXe
authored
Merge pull request #5 from tigrisdata/Xe/per-repo-bucket
feat(storage)!: per-keypair Tigris buckets, one per repo
2 parents 84f2e70 + e90cae8 commit 6a270b3

21 files changed

Lines changed: 1558 additions & 177 deletions

‎cmd/objgitd/example_hook_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ import (
1212
"time"
1313

1414
"github.com/go-git/go-billy/v6/memfs"
15-
"github.com/go-git/go-git/v6/plumbing/transport"
1615
"github.com/tigrisdata/objgit/internal/auth"
16+
"github.com/tigrisdata/objgit/internal/repofs"
1717
)
1818

1919
// TestExampleHookRuns pushes the repository's own example hook
@@ -37,8 +37,8 @@ func TestExampleHookRuns(t *testing.T) {
3737

3838
fs := memfs.New()
3939
d := &daemon{
40-
fs: fs,
41-
loader: transport.NewFilesystemLoader(fs, false),
40+
sysFS: fs,
41+
resolver: repofs.BucketResolver{Base: fs},
4242
authz: auth.AllowAnonymous{AllowWrite: true},
4343
allowHooks: true,
4444
hookTimeout: 30 * time.Second,
@@ -51,7 +51,7 @@ func TestExampleHookRuns(t *testing.T) {
5151
}
5252
go func() { _ = d.ServeGitProtocol(ctx, ln) }()
5353

54-
remote := "git://" + ln.Addr().String() + "/example.git"
54+
remote := "git://" + ln.Addr().String() + "/acme/example.git"
5555
work := t.TempDir()
5656
runGit(t, work, "init", "-b", "main")
5757
runGit(t, work, "config", "user.email", "test@example.com")

‎cmd/objgitd/git_protocol.go‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/go-git/go-git/v6/storage/filesystem"
1818
"github.com/tigrisdata/objgit/internal/auth"
1919
"github.com/tigrisdata/objgit/internal/metrics"
20+
"github.com/tigrisdata/objgit/internal/repofs"
2021
)
2122

2223
// handshakeTimeout bounds how long a client has to send its git-proto-request.
@@ -43,28 +44,42 @@ func (d *daemon) authorize(ctx context.Context, req auth.Request) auth.Decision
4344
return dec
4445
}
4546

46-
// daemon serves the git:// (TCP) protocol out of a billy filesystem.
47+
// daemon serves the git protocols out of billy filesystems resolved per repo.
4748
type daemon struct {
48-
fs billy.Filesystem
49-
loader transport.Loader
50-
authz auth.Authorizer
49+
// sysFS holds daemon-level state that is not scoped to a repository (the SSH
50+
// host key); repository storage is resolved per request via resolver.
51+
sysFS billy.Filesystem
52+
resolver repofs.Resolver
53+
authz auth.Authorizer
5154

5255
// allowHooks gates running .objgit/hooks/receive-pack after a push.
5356
allowHooks bool
5457
hookTimeout time.Duration
5558
}
5659

57-
// load opens the storer for repoPath and heals a dangling HEAD before returning
58-
// it (see ensureHEAD). It preserves the loader's error verbatim — notably
60+
// storerFor returns the bare-repository storer rooted at fs, or
61+
// transport.ErrRepositoryNotFound when no repository exists there. It reuses
62+
// go-git's own bare-repo detection (a "config" file at the root) by loading the
63+
// repository at the filesystem root.
64+
func storerFor(fs billy.Filesystem) (storage.Storer, error) {
65+
return transport.NewFilesystemLoader(fs, false).Load(&url.URL{Path: "/"})
66+
}
67+
68+
// load resolves the storer for ref and heals a dangling HEAD before returning it
69+
// (see ensureHEAD). It preserves storerFor's error verbatim — notably
5970
// transport.ErrRepositoryNotFound, which callers map to a 404 — and treats a
6071
// heal failure as non-fatal so a clone is never broken by a transient HEAD write.
61-
func (d *daemon) load(repoPath string) (storage.Storer, error) {
62-
st, err := d.loader.Load(&url.URL{Path: repoPath})
72+
func (d *daemon) load(ctx context.Context, ref repofs.RepoRef, cred repofs.Credential) (storage.Storer, error) {
73+
fs, err := d.resolver.Resolve(ctx, ref, cred, false)
74+
if err != nil {
75+
return nil, err
76+
}
77+
st, err := storerFor(fs)
6378
if err != nil {
6479
return nil, err
6580
}
6681
if err := ensureHEAD(st); err != nil {
67-
slog.Warn("could not repoint dangling HEAD", "path", repoPath, "err", err)
82+
slog.Warn("could not repoint dangling HEAD", "repo", ref.Path(), "err", err)
6883
}
6984
return st, nil
7085
}
@@ -140,29 +155,32 @@ func pickDefaultBranch(st storage.Storer) (plumbing.ReferenceName, error) {
140155
return first, nil
141156
}
142157

143-
// loadOrInit returns the storer for repoPath, creating an empty bare repository
144-
// on demand. Git's daemon never auto-creates; objgitd does, so a first push to
145-
// a new path just works.
146-
func (d *daemon) loadOrInit(repoPath string) (storage.Storer, error) {
147-
st, err := d.load(repoPath)
158+
// loadOrInit returns the storer for ref, creating an empty bare repository on
159+
// demand. Git's daemon never auto-creates; objgitd does, so a first push to a
160+
// new path just works.
161+
func (d *daemon) loadOrInit(ctx context.Context, ref repofs.RepoRef, cred repofs.Credential) (storage.Storer, error) {
162+
fs, err := d.resolver.Resolve(ctx, ref, cred, true)
163+
if err != nil {
164+
return nil, err
165+
}
166+
167+
st, err := storerFor(fs)
148168
if err == nil {
169+
if err := ensureHEAD(st); err != nil {
170+
slog.Warn("could not repoint dangling HEAD", "repo", ref.Path(), "err", err)
171+
}
149172
return st, nil
150173
}
151174
if !errors.Is(err, transport.ErrRepositoryNotFound) {
152175
return nil, err
153176
}
154177

155-
fs, err := d.fs.Chroot(repoPath)
156-
if err != nil {
157-
return nil, fmt.Errorf("chroot %q: %w", repoPath, err)
158-
}
159-
160178
st = filesystem.NewStorage(fs, cache.NewObjectLRUDefault())
161179
if _, err := git.Init(st, git.WithDefaultBranch(plumbing.NewBranchReferenceName("main"))); err != nil {
162180
return nil, fmt.Errorf("init bare repo: %w", err)
163181
}
164182

165183
metrics.ReposCreated()
166-
slog.Info("created repository", "path", repoPath)
184+
slog.Info("created repository", "repo", ref.Path())
167185
return st, nil
168186
}

‎cmd/objgitd/git_protocol_test.go‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/go-git/go-git/v6/plumbing/transport"
2121
"github.com/tigrisdata/objgit/internal/auth"
2222
"github.com/tigrisdata/objgit/internal/metrics"
23+
"github.com/tigrisdata/objgit/internal/repofs"
2324
)
2425

2526
// ServeGitProtocol accepts connections on l until ctx is cancelled or Accept fails.
@@ -71,6 +72,12 @@ func (d *daemon) handleGitProtocol(ctx context.Context, conn net.Conn) error {
7172
"remote", conn.RemoteAddr().String(),
7273
)
7374

75+
ref, err := repofs.Parse(req.Pathname)
76+
if err != nil {
77+
_, _ = pktline.WriteError(conn, err)
78+
return fmt.Errorf("invalid repo path %q: %w", req.Pathname, err)
79+
}
80+
7481
// ExtraParams carries e.g. "version=2"; transport.ProtocolVersion splits on ":".
7582
gitProtocol := strings.Join(req.ExtraParams, ":")
7683

@@ -83,7 +90,7 @@ func (d *daemon) handleGitProtocol(ctx context.Context, conn net.Conn) error {
8390
start := time.Now()
8491

8592
if d.authorize(ctx, auth.Request{
86-
Repo: req.Pathname,
93+
Repo: ref.Path(),
8794
Operation: operationFor(req.RequestCommand),
8895
Cred: auth.Anonymous{},
8996
Transport: "git",
@@ -93,7 +100,7 @@ func (d *daemon) handleGitProtocol(ctx context.Context, conn net.Conn) error {
93100
return fmt.Errorf("access denied for %q (%s)", req.Pathname, req.RequestCommand)
94101
}
95102

96-
err := d.serveGit(ctx, conn, r, req, gitProtocol)
103+
err = d.serveGit(ctx, conn, r, req, ref, gitProtocol)
97104
status := "ok"
98105
if err != nil {
99106
status = "error"
@@ -104,10 +111,10 @@ func (d *daemon) handleGitProtocol(ctx context.Context, conn net.Conn) error {
104111

105112
// serveGit dispatches a parsed, authorized git:// request to the matching
106113
// go-git transport command.
107-
func (d *daemon) serveGit(ctx context.Context, conn net.Conn, r io.ReadCloser, req packp.GitProtoRequest, gitProtocol string) error {
114+
func (d *daemon) serveGit(ctx context.Context, conn net.Conn, r io.ReadCloser, req packp.GitProtoRequest, ref repofs.RepoRef, gitProtocol string) error {
108115
switch req.RequestCommand {
109116
case transport.UploadPackService:
110-
st, err := d.load(req.Pathname)
117+
st, err := d.load(ctx, ref, repofs.Credential{})
111118
if err != nil {
112119
_, _ = pktline.WriteError(conn, fmt.Errorf("repository %q not found", req.Pathname))
113120
return fmt.Errorf("loading %q: %w", req.Pathname, err)
@@ -117,20 +124,20 @@ func (d *daemon) serveGit(ctx context.Context, conn net.Conn, r io.ReadCloser, r
117124
})
118125

119126
case transport.UploadArchiveService:
120-
st, err := d.load(req.Pathname)
127+
st, err := d.load(ctx, ref, repofs.Credential{})
121128
if err != nil {
122129
_, _ = pktline.WriteError(conn, fmt.Errorf("repository %q not found", req.Pathname))
123130
return fmt.Errorf("loading %q: %w", req.Pathname, err)
124131
}
125132
return transport.UploadArchive(ctx, st, r, conn, &transport.UploadArchiveRequest{})
126133

127134
case transport.ReceivePackService:
128-
st, err := d.loadOrInit(req.Pathname)
135+
st, err := d.loadOrInit(ctx, ref, repofs.Credential{})
129136
if err != nil {
130137
_, _ = pktline.WriteError(conn, fmt.Errorf("cannot open repository %q", req.Pathname))
131138
return fmt.Errorf("opening %q for push: %w", req.Pathname, err)
132139
}
133-
return d.receivePack(ctx, st, req.Pathname, r, conn, &transport.ReceivePackRequest{
140+
return d.receivePack(ctx, st, ref.Path(), r, conn, &transport.ReceivePackRequest{
134141
GitProtocol: gitProtocol,
135142
})
136143

@@ -150,9 +157,9 @@ func TestDaemonPushCreatesRepo(t *testing.T) {
150157

151158
fs := memfs.New()
152159
d := &daemon{
153-
fs: fs,
154-
loader: transport.NewFilesystemLoader(fs, false),
155-
authz: auth.AllowAnonymous{AllowWrite: true},
160+
sysFS: fs,
161+
resolver: repofs.BucketResolver{Base: fs},
162+
authz: auth.AllowAnonymous{AllowWrite: true},
156163
}
157164

158165
ctx, cancel := context.WithCancel(context.Background())
@@ -166,7 +173,7 @@ func TestDaemonPushCreatesRepo(t *testing.T) {
166173
srvErr := make(chan error, 1)
167174
go func() { srvErr <- d.ServeGitProtocol(ctx, ln) }()
168175

169-
remote := "git://" + ln.Addr().String() + "/test.git"
176+
remote := "git://" + ln.Addr().String() + "/acme/test.git"
170177

171178
work := t.TempDir()
172179
runGit(t, work, "init", "-b", "main")
@@ -177,8 +184,8 @@ func TestDaemonPushCreatesRepo(t *testing.T) {
177184
// The repository does not exist yet; the push must create it.
178185
runGit(t, work, "push", remote, "main")
179186

180-
if _, err := fs.Stat("/test.git/config"); err != nil {
181-
t.Fatalf("expected bare repo to be created on push, but %q is missing: %v", "/test.git/config", err)
187+
if _, err := fs.Stat("/acme/test/config"); err != nil {
188+
t.Fatalf("expected bare repo to be created on push, but %q is missing: %v", "/acme/test/config", err)
182189
}
183190

184191
// Round-trip: a clone must recover the pushed commit.
@@ -207,9 +214,9 @@ func TestDaemonPushDisabled(t *testing.T) {
207214

208215
fs := memfs.New()
209216
d := &daemon{
210-
fs: fs,
211-
loader: transport.NewFilesystemLoader(fs, false),
212-
authz: auth.AllowAnonymous{AllowWrite: false},
217+
sysFS: fs,
218+
resolver: repofs.BucketResolver{Base: fs},
219+
authz: auth.AllowAnonymous{AllowWrite: false},
213220
}
214221

215222
ctx, cancel := context.WithCancel(context.Background())
@@ -221,7 +228,7 @@ func TestDaemonPushDisabled(t *testing.T) {
221228
}
222229
go func() { _ = d.ServeGitProtocol(ctx, ln) }()
223230

224-
remote := "git://" + ln.Addr().String() + "/test.git"
231+
remote := "git://" + ln.Addr().String() + "/acme/test.git"
225232

226233
work := t.TempDir()
227234
runGit(t, work, "init", "-b", "main")
@@ -233,7 +240,7 @@ func TestDaemonPushDisabled(t *testing.T) {
233240
t.Fatalf("expected push to be rejected when allowPush is false, got success:\n%s", out)
234241
}
235242

236-
if _, err := fs.Stat("/test.git/config"); err == nil {
243+
if _, err := fs.Stat("/acme/test/config"); err == nil {
237244
t.Fatal("repository must not be created when push is disabled")
238245
}
239246
}
@@ -251,9 +258,9 @@ func TestDaemonPushKeepsPack(t *testing.T) {
251258

252259
fs := memfs.New()
253260
d := &daemon{
254-
fs: fs,
255-
loader: transport.NewFilesystemLoader(fs, false),
256-
authz: auth.AllowAnonymous{AllowWrite: true},
261+
sysFS: fs,
262+
resolver: repofs.BucketResolver{Base: fs},
263+
authz: auth.AllowAnonymous{AllowWrite: true},
257264
}
258265

259266
ctx, cancel := context.WithCancel(context.Background())
@@ -265,7 +272,7 @@ func TestDaemonPushKeepsPack(t *testing.T) {
265272
}
266273
go func() { _ = d.ServeGitProtocol(ctx, ln) }()
267274

268-
remote := "git://" + ln.Addr().String() + "/test.git"
275+
remote := "git://" + ln.Addr().String() + "/acme/test.git"
269276

270277
work := t.TempDir()
271278
runGit(t, work, "init", "-b", "main")
@@ -276,7 +283,7 @@ func TestDaemonPushKeepsPack(t *testing.T) {
276283
runGit(t, work, "commit", "-m", "initial") // blob + tree + commit
277284
runGit(t, work, "push", remote, "main")
278285

279-
assertPackedRepo(t, fs, "/test.git")
286+
assertPackedRepo(t, fs, "/acme/test")
280287
}
281288

282289
// assertPackedRepo fails unless repoPath holds at least one packfile and no loose

‎cmd/objgitd/head_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ import (
1313
"github.com/go-git/go-billy/v6/memfs"
1414
"github.com/go-git/go-git/v6/plumbing"
1515
"github.com/go-git/go-git/v6/plumbing/cache"
16-
"github.com/go-git/go-git/v6/plumbing/transport"
1716
"github.com/go-git/go-git/v6/storage/filesystem"
1817
"github.com/tigrisdata/objgit/internal/auth"
18+
"github.com/tigrisdata/objgit/internal/repofs"
1919
)
2020

2121
// dummyHash is a stand-in object id for branch refs in unit tests; ensureHEAD
@@ -148,11 +148,11 @@ func TestSmartHTTPHealsDanglingHEAD(t *testing.T) {
148148
}
149149

150150
fs := memfs.New()
151-
ts := httptest.NewServer(&daemon{
152-
fs: fs,
153-
loader: transport.NewFilesystemLoader(fs, false),
154-
authz: auth.AllowAnonymous{AllowWrite: true},
155-
})
151+
ts := httptest.NewServer((&daemon{
152+
sysFS: fs,
153+
resolver: repofs.BucketResolver{Base: fs},
154+
authz: auth.AllowAnonymous{AllowWrite: true},
155+
}).httpHandler())
156156
t.Cleanup(ts.Close)
157157

158158
// Push a single "master" branch (no "main"), like a project whose default
@@ -164,18 +164,18 @@ func TestSmartHTTPHealsDanglingHEAD(t *testing.T) {
164164
writeFile(t, filepath.Join(work, "README.md"), "hello\n")
165165
runGit(t, work, "add", ".")
166166
runGit(t, work, "commit", "-m", "initial")
167-
if out, err := tryGit(work, "push", ts.URL+"/go.git", "master"); err != nil {
167+
if out, err := tryGit(work, "push", ts.URL+"/acme/go.git", "master"); err != nil {
168168
t.Fatalf("push failed: %v\n%s", err, out)
169169
}
170170

171171
// Re-break HEAD to simulate a repo created before this fix (post-push heal
172172
// would otherwise have already fixed it): point HEAD back at the dangling
173173
// refs/heads/main directly in the backing store. The very next load (this
174174
// clone) must heal it on the way to serving the advertisement.
175-
breakHEAD(t, fs, "/go.git")
175+
breakHEAD(t, fs, "/acme/go")
176176

177177
dst := t.TempDir()
178-
out, err := tryGit(dst, "clone", ts.URL+"/go.git", "cloned")
178+
out, err := tryGit(dst, "clone", ts.URL+"/acme/go.git", "cloned")
179179
if err != nil {
180180
t.Fatalf("clone failed: %v\n%s", err, out)
181181
}
@@ -195,7 +195,7 @@ func TestSmartHTTPHealsDanglingHEAD(t *testing.T) {
195195
}
196196

197197
// After a load-healed clone, the advertisement now carries the symref.
198-
if body := getInfoRefs(t, ts.URL+"/go.git"); !strings.Contains(body, "symref=HEAD:refs/heads/master") {
198+
if body := getInfoRefs(t, ts.URL+"/acme/go.git"); !strings.Contains(body, "symref=HEAD:refs/heads/master") {
199199
t.Errorf("expected symref=HEAD:refs/heads/master after heal; advertisement:\n%q", body)
200200
}
201201
}

0 commit comments

Comments
 (0)