Skip to content

Commit 01dc845

Browse files
KarthikNayakgitster
authored andcommitted
refs: allow reference location in refstorage config
The 'extensions.refStorage' config is used to specify the reference backend for a given repository. Both the 'files' and 'reftable' backends utilize the $GIT_DIR as the reference folder by default in `get_main_ref_store()`. Since the reference backends are pluggable, this means that they could work with out-of-tree reference directories too. Extend the 'refStorage' config to also support taking an URI input, where users can specify the reference backend and the location. Add the required changes to obtain and propagate this value to the individual backends. Add the necessary documentation and tests. Traditionally, for linked worktrees, references were stored in the '$GIT_DIR/worktrees/<wt_id>' path. But when using an alternate reference storage path, it doesn't make sense to store the main worktree references in the new path, and the linked worktree references in the $GIT_DIR. So, let's store linked worktree references in '$ALTERNATE_REFERENCE_DIR/worktrees/<wt_id>'. To do this, create the necessary files and folders while also adding stubs in the $GIT_DIR path to ensure that it is still considered a Git directory. Ideally, we would want to pass in a `struct worktree *` to individual backends, instead of passing the `gitdir`. This allows them to handle worktree specific logic. Currently, that is not possible since the worktree code is: - Tied to using the global `the_repository` variable. - Is not setup before the reference database during initialization of the repository. Add a TODO in 'refs.c' to ensure we can eventually make that change. Helped-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Karthik Nayak <karthik.188@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent d74aacd commit 01dc845

9 files changed

Lines changed: 259 additions & 9 deletions

File tree

Documentation/config/extensions.adoc

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,24 @@ For historical reasons, this extension is respected regardless of the
5757
`core.repositoryFormatVersion` setting.
5858
5959
refStorage:::
60-
Specify the ref storage format to use. The acceptable values are:
60+
Specify the ref storage format and a corresponding payload. The value
61+
can be either a format name or a URI:
6162
+
6263
--
64+
* A format name alone (e.g., `reftable` or `files`).
65+
66+
* A URI format `<format>://<payload>` explicitly specifies both the
67+
format and payload (e.g., `reftable:///foo/bar`).
68+
69+
Supported format names are:
70+
6371
include::../ref-storage-format.adoc[]
72+
73+
The payload is passed directly to the reference backend. For the files and
74+
reftable backends, this must be a filesystem path where the references will
75+
be stored. Defaulting to the commondir when no payload is provided. Relative
76+
paths are resolved relative to the `$GIT_DIR`. Future backends may support
77+
other payload schemes, e.g., postgres://127.0.0.1:5432?database=myrepo.
6478
--
6579
+
6680
Note that this setting should only be set by linkgit:git-init[1] or

builtin/worktree.c

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,39 @@ static int make_worktree_orphan(const char * ref, const struct add_opts *opts,
425425
return run_command(&cp);
426426
}
427427

428+
/*
429+
* References for worktrees are generally stored in '$GIT_DIR/worktrees/<wt_id>'.
430+
* But when using alternate reference directories, we want to store the worktree
431+
* references in '$ALTERNATE_REFERENCE_DIR/worktrees/<wt_id>'.
432+
*
433+
* Create the necessary folder structure to facilitate the same. But to ensure
434+
* that the former path is still considered a Git directory, add stubs.
435+
*/
436+
static void setup_alternate_ref_dir(struct worktree *wt, const char *wt_git_path)
437+
{
438+
struct strbuf sb = STRBUF_INIT;
439+
char *path;
440+
441+
path = wt->repo->ref_storage_payload;
442+
if (!path)
443+
return;
444+
445+
if (!is_absolute_path(path))
446+
strbuf_addf(&sb, "%s/", wt->repo->commondir);
447+
448+
strbuf_addf(&sb, "%s/worktrees", path);
449+
safe_create_dir(wt->repo, sb.buf, 1);
450+
strbuf_addf(&sb, "/%s", wt->id);
451+
safe_create_dir(wt->repo, sb.buf, 1);
452+
strbuf_reset(&sb);
453+
454+
strbuf_addf(&sb, "this worktree stores references in %s/worktrees/%s",
455+
path, wt->id);
456+
refs_create_refdir_stubs(wt->repo, wt_git_path, sb.buf);
457+
458+
strbuf_release(&sb);
459+
}
460+
428461
static int add_worktree(const char *path, const char *refname,
429462
const struct add_opts *opts)
430463
{
@@ -518,6 +551,7 @@ static int add_worktree(const char *path, const char *refname,
518551
ret = error(_("could not find created worktree '%s'"), name);
519552
goto done;
520553
}
554+
setup_alternate_ref_dir(wt, sb_repo.buf);
521555
wt_refs = get_worktree_ref_store(wt);
522556

523557
ret = ref_store_create_on_disk(wt_refs, REF_STORE_CREATE_ON_DISK_IS_WORKTREE, &sb);

refs.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2291,7 +2291,11 @@ static struct ref_store *ref_store_init(struct repository *repo,
22912291
if (!be)
22922292
BUG("reference backend is unknown");
22932293

2294-
refs = be->init(repo, NULL, gitdir, flags);
2294+
/*
2295+
* TODO Send in a 'struct worktree' instead of a 'gitdir', and
2296+
* allow the backend to handle how it wants to deal with worktrees.
2297+
*/
2298+
refs = be->init(repo, repo->ref_storage_payload, gitdir, flags);
22952299
return refs;
22962300
}
22972301

repository.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -193,9 +193,12 @@ void repo_set_compat_hash_algo(struct repository *repo, int algo)
193193
}
194194

195195
void repo_set_ref_storage_format(struct repository *repo,
196-
enum ref_storage_format format)
196+
enum ref_storage_format format,
197+
const char *payload)
197198
{
198199
repo->ref_storage_format = format;
200+
free(repo->ref_storage_payload);
201+
repo->ref_storage_payload = xstrdup_or_null(payload);
199202
}
200203

201204
/*
@@ -277,7 +280,8 @@ int repo_init(struct repository *repo,
277280

278281
repo_set_hash_algo(repo, format.hash_algo);
279282
repo_set_compat_hash_algo(repo, format.compat_hash_algo);
280-
repo_set_ref_storage_format(repo, format.ref_storage_format);
283+
repo_set_ref_storage_format(repo, format.ref_storage_format,
284+
format.ref_storage_payload);
281285
repo->repository_format_worktree_config = format.worktree_config;
282286
repo->repository_format_relative_worktrees = format.relative_worktrees;
283287
repo->repository_format_precious_objects = format.precious_objects;
@@ -369,6 +373,7 @@ void repo_clear(struct repository *repo)
369373
FREE_AND_NULL(repo->index_file);
370374
FREE_AND_NULL(repo->worktree);
371375
FREE_AND_NULL(repo->submodule_prefix);
376+
FREE_AND_NULL(repo->ref_storage_payload);
372377

373378
odb_free(repo->objects);
374379
repo->objects = NULL;

repository.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,11 @@ struct repository {
150150

151151
/* Repository's reference storage format, as serialized on disk. */
152152
enum ref_storage_format ref_storage_format;
153+
/*
154+
* Reference storage information as needed for the backend. This contains
155+
* only the payload from the reference URI without the schema.
156+
*/
157+
char *ref_storage_payload;
153158

154159
/* A unique-id for tracing purposes. */
155160
int trace2_repo_id;
@@ -204,7 +209,8 @@ void repo_set_worktree(struct repository *repo, const char *path);
204209
void repo_set_hash_algo(struct repository *repo, int algo);
205210
void repo_set_compat_hash_algo(struct repository *repo, int compat_algo);
206211
void repo_set_ref_storage_format(struct repository *repo,
207-
enum ref_storage_format format);
212+
enum ref_storage_format format,
213+
const char *payload);
208214
void initialize_repository(struct repository *repo);
209215
RESULT_MUST_BE_USED
210216
int repo_init(struct repository *r, const char *gitdir, const char *worktree);

setup.c

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -632,6 +632,21 @@ static enum extension_result handle_extension_v0(const char *var,
632632
return EXTENSION_UNKNOWN;
633633
}
634634

635+
static void parse_reference_uri(const char *value, char **format,
636+
char **payload)
637+
{
638+
const char *schema_end;
639+
640+
schema_end = strstr(value, "://");
641+
if (!schema_end) {
642+
*format = xstrdup(value);
643+
*payload = NULL;
644+
} else {
645+
*format = xstrndup(value, schema_end - value);
646+
*payload = xstrdup_or_null(schema_end + 3);
647+
}
648+
}
649+
635650
/*
636651
* Record any new extensions in this function.
637652
*/
@@ -674,10 +689,17 @@ static enum extension_result handle_extension(const char *var,
674689
return EXTENSION_OK;
675690
} else if (!strcmp(ext, "refstorage")) {
676691
unsigned int format;
692+
char *format_str;
677693

678694
if (!value)
679695
return config_error_nonbool(var);
680-
format = ref_storage_format_by_name(value);
696+
697+
parse_reference_uri(value, &format_str,
698+
&data->ref_storage_payload);
699+
700+
format = ref_storage_format_by_name(format_str);
701+
free(format_str);
702+
681703
if (format == REF_STORAGE_FORMAT_UNKNOWN)
682704
return error(_("invalid value for '%s': '%s'"),
683705
"extensions.refstorage", value);
@@ -850,6 +872,7 @@ void clear_repository_format(struct repository_format *format)
850872
string_list_clear(&format->v1_only_extensions, 0);
851873
free(format->work_tree);
852874
free(format->partial_clone);
875+
free(format->ref_storage_payload);
853876
init_repository_format(format);
854877
}
855878

@@ -1942,7 +1965,8 @@ const char *setup_git_directory_gently(int *nongit_ok)
19421965
repo_set_compat_hash_algo(the_repository,
19431966
repo_fmt.compat_hash_algo);
19441967
repo_set_ref_storage_format(the_repository,
1945-
repo_fmt.ref_storage_format);
1968+
repo_fmt.ref_storage_format,
1969+
repo_fmt.ref_storage_payload);
19461970
the_repository->repository_format_worktree_config =
19471971
repo_fmt.worktree_config;
19481972
the_repository->repository_format_relative_worktrees =
@@ -2042,7 +2066,8 @@ void check_repository_format(struct repository_format *fmt)
20422066
repo_set_hash_algo(the_repository, fmt->hash_algo);
20432067
repo_set_compat_hash_algo(the_repository, fmt->compat_hash_algo);
20442068
repo_set_ref_storage_format(the_repository,
2045-
fmt->ref_storage_format);
2069+
fmt->ref_storage_format,
2070+
fmt->ref_storage_payload);
20462071
the_repository->repository_format_worktree_config =
20472072
fmt->worktree_config;
20482073
the_repository->repository_format_relative_worktrees =
@@ -2643,7 +2668,8 @@ static void repository_format_configure(struct repository_format *repo_fmt,
26432668
} else {
26442669
repo_fmt->ref_storage_format = REF_STORAGE_FORMAT_DEFAULT;
26452670
}
2646-
repo_set_ref_storage_format(the_repository, repo_fmt->ref_storage_format);
2671+
repo_set_ref_storage_format(the_repository, repo_fmt->ref_storage_format,
2672+
repo_fmt->ref_storage_payload);
26472673
}
26482674

26492675
int init_db(const char *git_dir, const char *real_git_dir,

setup.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ struct repository_format {
171171
int hash_algo;
172172
int compat_hash_algo;
173173
enum ref_storage_format ref_storage_format;
174+
char *ref_storage_payload;
174175
int sparse_index;
175176
char *work_tree;
176177
struct string_list unknown_extensions;

t/meson.build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ integration_tests = [
210210
't1420-lost-found.sh',
211211
't1421-reflog-write.sh',
212212
't1422-show-ref-exists.sh',
213+
't1423-ref-backend.sh',
213214
't1430-bad-ref-name.sh',
214215
't1450-fsck.sh',
215216
't1451-fsck-buffer.sh',

t/t1423-ref-backend.sh

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#!/bin/sh
2+
3+
test_description='Test reference backend URIs'
4+
5+
. ./test-lib.sh
6+
7+
# Run a git command with the provided reference storage. Reset the backend
8+
# post running the command.
9+
# Usage: run_with_uri <repo> <backend> <uri> <cmd>
10+
# <repo> is the relative path to the repo to run the command in.
11+
# <backend> is the original ref storage of the repo.
12+
# <uri> is the new URI to be set for the ref storage.
13+
# <cmd> is the git subcommand to be run in the repository.
14+
run_with_uri () {
15+
repo=$1 &&
16+
backend=$2 &&
17+
uri=$3 &&
18+
cmd=$4 &&
19+
20+
git -C "$repo" config set core.repositoryformatversion 1
21+
git -C "$repo" config set extensions.refStorage "$uri" &&
22+
git -C "$repo" $cmd &&
23+
git -C "$repo" config set extensions.refStorage "$backend"
24+
}
25+
26+
# Test a repository with a given reference storage by running and comparing
27+
# 'git refs list' before and after setting the new reference backend. If
28+
# err_msg is set, expect the command to fail and grep for the provided err_msg.
29+
# Usage: run_with_uri <repo> <backend> <uri> <cmd>
30+
# <repo> is the relative path to the repo to run the command in.
31+
# <backend> is the original ref storage of the repo.
32+
# <uri> is the new URI to be set for the ref storage.
33+
# <err_msg> (optional) if set, check if 'git-refs(1)' failed with the provided msg.
34+
test_refs_backend () {
35+
repo=$1 &&
36+
backend=$2 &&
37+
uri=$3 &&
38+
err_msg=$4 &&
39+
40+
git -C "$repo" config set core.repositoryformatversion 1 &&
41+
if test -n "$err_msg";
42+
then
43+
git -C "$repo" config set extensions.refStorage "$uri" &&
44+
test_must_fail git -C "$repo" refs list 2>err &&
45+
test_grep "$err_msg" err
46+
else
47+
git -C "$repo" refs list >expect &&
48+
run_with_uri "$repo" "$backend" "$uri" "refs list" >actual &&
49+
test_cmp expect actual
50+
fi
51+
}
52+
53+
test_expect_success 'URI is invalid' '
54+
test_when_finished "rm -rf repo" &&
55+
git init repo &&
56+
test_refs_backend repo files "reftable@/home/reftable" \
57+
"invalid value for ${SQ}extensions.refstorage${SQ}"
58+
'
59+
60+
test_expect_success 'URI ends with colon' '
61+
test_when_finished "rm -rf repo" &&
62+
git init repo &&
63+
test_refs_backend repo files "reftable:" \
64+
"invalid value for ${SQ}extensions.refstorage${SQ}"
65+
'
66+
67+
test_expect_success 'unknown reference backend' '
68+
test_when_finished "rm -rf repo" &&
69+
git init repo &&
70+
test_refs_backend repo files "db://.git" \
71+
"invalid value for ${SQ}extensions.refstorage${SQ}"
72+
'
73+
74+
ref_formats="files reftable"
75+
for from_format in $ref_formats
76+
do
77+
78+
for to_format in $ref_formats
79+
do
80+
if test "$from_format" = "$to_format"
81+
then
82+
continue
83+
fi
84+
85+
86+
for dir in "$(pwd)/repo/.git" "."
87+
do
88+
89+
test_expect_success "read from $to_format backend, $dir dir" '
90+
test_when_finished "rm -rf repo" &&
91+
git init --ref-format=$from_format repo &&
92+
(
93+
cd repo &&
94+
test_commit 1 &&
95+
test_commit 2 &&
96+
test_commit 3 &&
97+
98+
git refs migrate --dry-run --ref-format=$to_format >out &&
99+
BACKEND_PATH="$dir/$(sed "s/.* ${SQ}.git\/\(.*\)${SQ}/\1/" out)" &&
100+
test_refs_backend . $from_format "$to_format://$BACKEND_PATH" "$method"
101+
)
102+
'
103+
104+
test_expect_success "write to $to_format backend, $dir dir" '
105+
test_when_finished "rm -rf repo" &&
106+
git init --ref-format=$from_format repo &&
107+
(
108+
cd repo &&
109+
test_commit 1 &&
110+
test_commit 2 &&
111+
test_commit 3 &&
112+
113+
git refs migrate --dry-run --ref-format=$to_format >out &&
114+
BACKEND_PATH="$dir/$(sed "s/.* ${SQ}.git\/\(.*\)${SQ}/\1/" out)" &&
115+
116+
test_refs_backend . $from_format "$to_format://$BACKEND_PATH" &&
117+
118+
git refs list >expect &&
119+
run_with_uri . "$from_format" "$to_format://$BACKEND_PATH" "tag -d 1" &&
120+
git refs list >actual &&
121+
test_cmp expect actual &&
122+
123+
git refs list | grep -v "refs/tags/1" >expect &&
124+
run_with_uri . "$from_format" "$to_format://$BACKEND_PATH" "refs list" >actual &&
125+
test_cmp expect actual
126+
)
127+
'
128+
129+
test_expect_success "with worktree and $to_format backend, $dir dir" '
130+
test_when_finished "rm -rf repo wt" &&
131+
git init --ref-format=$from_format repo &&
132+
(
133+
cd repo &&
134+
test_commit 1 &&
135+
test_commit 2 &&
136+
test_commit 3 &&
137+
138+
git refs migrate --dry-run --ref-format=$to_format >out &&
139+
BACKEND_PATH="$dir/$(sed "s/.* ${SQ}.git\/\(.*\)${SQ}/\1/" out)" &&
140+
141+
git config set core.repositoryformatversion 1 &&
142+
git config set extensions.refStorage "$to_format://$BACKEND_PATH" &&
143+
144+
git worktree add ../wt 2
145+
) &&
146+
147+
git -C repo for-each-ref --include-root-refs >expect &&
148+
git -C wt for-each-ref --include-root-refs >expect &&
149+
! test_cmp expect actual &&
150+
151+
git -C wt rev-parse 2 >expect &&
152+
git -C wt rev-parse HEAD >actual &&
153+
test_cmp expect actual
154+
'
155+
done # closes dir
156+
done # closes to_format
157+
done # closes from_format
158+
159+
test_done

0 commit comments

Comments
 (0)