multi-pack-index: implement --max-chain-depth

Signed-off-by: Vicent Marti <vmg@strn.cat>

Author: vmg

Documentation/git-multi-pack-index.adoc

@@ -15,7 +15,7 @@ SYNOPSIS
 			 [--base=<checksum>]
 'git multi-pack-index' [<options>] compact [--[no-]incremental]
 		         [--[no-]bitmap] [--base=<checksum>] [--[no-]write-chain-file]
-			 <from> <to>
+			 [--max-chain-depth=<n> [--split-factor=<n>]] [<from> <to>]
 'git multi-pack-index' [<options>] verify
 'git multi-pack-index' [<options>] expire
 'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -125,6 +125,19 @@ compact::
 		as the base for the compacted result, instead of using
 		the immediate parent of `<from>`. The special value
 		`none` indicates that the result should have no base.
+
+	--max-chain-depth=<n>::
+		Automatically choose a MIDX layer range to compact when the
+		current incremental MIDX chain has more than `<n>` layers.
+		If the current chain has at most `<n>` layers, this option
+		exits without writing a new MIDX. Otherwise, adjacent layers
+		are selected from the chain tip according to
+		`--split-factor=<n>` and compacted into a new incremental
+		layer.
+
+	--split-factor=<n>::
+		With `--max-chain-depth`, control how many adjacent layers
+		are merged. Defaults to 2.
 --
 +
 Note that the compact command requires writing a version-2 midx that

builtin/multi-pack-index.c

@@ -4,6 +4,7 @@
 #include "config.h"
 #include "environment.h"
 #include "gettext.h"
+#include "parse.h"
 #include "parse-options.h"
 #include "midx.h"
 #include "strbuf.h"
@@ -22,7 +23,7 @@
 #define BUILTIN_MIDX_COMPACT_USAGE \
 	N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
 	   "  [--[no-]bitmap] [--base=<checksum>] [--[no-]write-chain-file]\n" \
-	   "  <from> <to>")
+	   "  [--max-chain-depth=<n> [--split-factor=<n>]] [<from> <to>]")
 
 #define BUILTIN_MIDX_VERIFY_USAGE \
 	N_("git multi-pack-index [<options>] verify")
@@ -68,6 +69,8 @@ static struct opts_multi_pack_index {
 	const char *incremental_base;
 	char *refs_snapshot;
 	unsigned long batch_size;
+	unsigned long max_chain_depth;
+	unsigned long split_factor;
 	unsigned flags;
 	int stdin_packs;
 } opts;
@@ -143,6 +146,20 @@ static void read_packs_from_stdin(struct string_list *to)
 	strbuf_release(&buf);
 }
 
+static int parse_positive_ulong(const struct option *opt, const char *arg,
+				int unset)
+{
+	unsigned long *value = opt->value;
+
+	if (unset) {
+		*value = 0;
+		return 0;
+	}
+	if (!arg || !git_parse_ulong(arg, value) || !*value)
+		return error(_("%s must be greater than zero"), opt->long_name);
+	return 0;
+}
+
 static int cmd_multi_pack_index_write(int argc, const char **argv,
 				      const char *prefix,
 				      struct repository *repo)
@@ -251,6 +268,13 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 		OPT_NEGBIT(0, "write-chain-file", &opts.flags,
 			N_("write the multi-pack-index chain file"),
 			MIDX_WRITE_NO_CHAIN),
+		OPT_CALLBACK(0, "max-chain-depth", &opts.max_chain_depth,
+			     N_("n"),
+			     N_("automatically compact when the chain has more than this many layers"),
+			     parse_positive_ulong),
+		OPT_CALLBACK(0, "split-factor", &opts.split_factor, N_("n"),
+			     N_("with --max-chain-depth, compact adjacent layers according to this split factor"),
+			     parse_positive_ulong),
 		OPT_END(),
 	};
 
@@ -266,9 +290,18 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 			     options, builtin_multi_pack_index_compact_usage,
 			     0);
 
-	if (argc != 2)
+	if (opts.max_chain_depth) {
+		if (argc)
+			usage_with_options(builtin_multi_pack_index_compact_usage,
+					   options);
+	} else if (argc != 2) {
 		usage_with_options(builtin_multi_pack_index_compact_usage,
 				   options);
+	} else if (opts.split_factor) {
+		error(_("cannot use --split-factor without --max-chain-depth"));
+		usage_with_options(builtin_multi_pack_index_compact_usage,
+				   options);
+	}
 
 	if (opts.flags & MIDX_WRITE_NO_CHAIN &&
 	    !(opts.flags & MIDX_WRITE_INCREMENTAL)) {
@@ -278,10 +311,38 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 				   options);
 	}
 
+	if (opts.max_chain_depth) {
+		if (opts.incremental_base) {
+			error(_("cannot use --max-chain-depth with --base"));
+			usage_with_options(builtin_multi_pack_index_compact_usage,
+					   options);
+		}
+		if (opts.flags & MIDX_WRITE_NO_CHAIN) {
+			error(_("cannot use --max-chain-depth with --no-write-chain-file"));
+			usage_with_options(builtin_multi_pack_index_compact_usage,
+					   options);
+		}
+		if (!opts.split_factor)
+			opts.split_factor = 2;
+		if (opts.max_chain_depth > UINT32_MAX ||
+		    opts.split_factor > UINT32_MAX) {
+			error(_("--max-chain-depth and --split-factor must fit in 32 bits"));
+			usage_with_options(builtin_multi_pack_index_compact_usage,
+					   options);
+		}
+		opts.flags |= MIDX_WRITE_INCREMENTAL;
+	}
+
 	source = handle_object_dir_option(the_repository);
 
 	FREE_AND_NULL(options);
 
+	if (opts.max_chain_depth)
+		return compact_midx_chain_auto(source,
+					       (uint32_t)opts.max_chain_depth,
+					       (uint32_t)opts.split_factor,
+					       opts.flags);
+
 	m = get_multi_pack_index(source);
 
 	for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) {

midx-write.c

@@ -1898,6 +1898,79 @@ int write_midx_file_compact(struct odb_source *source,
 	return write_midx_internal(&opts);
 }
 
+int compact_midx_chain_auto(struct odb_source *source,
+			    uint32_t max_chain_depth,
+			    uint32_t split_factor,
+			    unsigned flags)
+{
+	struct repository *r = source->odb->repo;
+	struct multi_pack_index *m;
+	struct multi_pack_index *from_midx, *to_midx, *next;
+	uint32_t depth = 0;
+	uint32_t compact_objects;
+
+	if (!max_chain_depth)
+		return error(_("--max-chain-depth must be greater than zero"));
+	if (!split_factor)
+		return error(_("--split-factor must be greater than zero"));
+
+	trace2_region_enter("midx", "compact_midx_chain_auto", r);
+
+	odb_reprepare(r->objects);
+	m = get_multi_pack_index(source);
+	for (struct multi_pack_index *cur = m; cur; cur = cur->base_midx)
+		depth++;
+
+	trace2_data_intmax("midx", r, "auto:chain_depth", depth);
+	trace2_data_intmax("midx", r, "auto:max_chain_depth", max_chain_depth);
+	trace2_data_intmax("midx", r, "auto:split_factor", split_factor);
+
+	if (depth <= max_chain_depth) {
+		trace2_data_string("midx", r, "auto:skip", "chain-depth");
+		trace2_region_leave("midx", "compact_midx_chain_auto", r);
+		return 0;
+	}
+	if (!m) {
+		trace2_data_string("midx", r, "auto:skip", "no-midx");
+		trace2_region_leave("midx", "compact_midx_chain_auto", r);
+		return 0;
+	}
+
+	to_midx = m;
+	from_midx = m;
+	compact_objects = m->num_objects;
+	while ((next = from_midx->base_midx)) {
+		if (compact_objects < next->num_objects / split_factor)
+			break;
+		if (unsigned_add_overflows(compact_objects, next->num_objects)) {
+			trace2_region_leave("midx", "compact_midx_chain_auto", r);
+			return error(_("too many objects in automatic MIDX compaction"));
+		}
+		compact_objects += next->num_objects;
+		from_midx = next;
+	}
+
+	if (from_midx == to_midx) {
+		trace2_data_string("midx", r, "auto:skip", "split-factor");
+		trace2_region_leave("midx", "compact_midx_chain_auto", r);
+		return 0;
+	}
+
+	trace2_data_string("midx", r, "auto:from", midx_get_checksum_hex(from_midx));
+	trace2_data_string("midx", r, "auto:to", midx_get_checksum_hex(to_midx));
+	trace2_data_intmax("midx", r, "auto:objects", compact_objects);
+
+	flags |= MIDX_WRITE_INCREMENTAL;
+	/*
+	 * The automatic range selector only chooses an existing contiguous
+	 * suffix of the chain, so the compacted layer naturally uses from's
+	 * immediate base as its base. Custom --base is for explicit callers.
+	 */
+	int ret = write_midx_file_compact(source, from_midx, to_midx, NULL, flags);
+	trace2_region_leave("midx", "compact_midx_chain_auto", r);
+	return ret;
+}
+
 int expire_midx_packs(struct odb_source *source, unsigned flags)
 {
 	uint32_t i, *count, result = 0;

midx.h

@@ -143,6 +143,10 @@ int write_midx_file_compact(struct odb_source *source,
 			    struct multi_pack_index *to,
 			    const char *incremental_base,
 			    unsigned flags);
+int compact_midx_chain_auto(struct odb_source *source,
+			    uint32_t max_chain_depth,
+			    uint32_t split_factor,
+			    unsigned flags);
 void clear_midx_file(struct repository *r);
 void clear_incremental_midx_files(struct repository *r,
 				  const struct strvec *keep_hashes);

t/t5335-compact-multi-pack-index.sh

@@ -403,4 +403,61 @@ test_expect_success 'MIDX compaction with bogus --base checksum' '
 	)
 '
 
+test_expect_success 'automatic MIDX compaction no-ops below chain depth cap' '
+	git init midx-compact-auto-noop &&
+	(
+		cd midx-compact-auto-noop &&
+
+		git config maintenance.auto false &&
+
+		write_packs A B C &&
+		cp "$midx_chain" "$midx_chain.bak" &&
+
+		git multi-pack-index compact --max-chain-depth=3 &&
+		test_cmp "$midx_chain.bak" "$midx_chain"
+	)
+'
+
+test_expect_success 'automatic MIDX compaction picks and compacts a range' '
+	git init midx-compact-auto &&
+	(
+		cd midx-compact-auto &&
+
+		git config maintenance.auto false &&
+
+		write_packs A B C D E &&
+		test_line_count = 5 "$midx_chain" &&
+
+		git multi-pack-index compact --bitmap \
+			--max-chain-depth=3 --split-factor=2 &&
+		test_line_count = 1 "$midx_chain" &&
+		test_path_is_file "$midxdir/multi-pack-index-$(nth_line 1 "$midx_chain").bitmap" &&
+		git multi-pack-index verify &&
+		test_midx_layer_object_uniqueness
+	)
+'
+
+test_expect_success 'automatic MIDX compaction rejects invalid knobs' '
+	git init midx-compact-auto-invalid &&
+	(
+		cd midx-compact-auto-invalid &&
+
+		write_packs A B &&
+
+		test_must_fail git multi-pack-index compact \
+			--max-chain-depth=0 2>err &&
+		test_grep "max-chain-depth must be greater than zero" err &&
+
+		test_must_fail git multi-pack-index compact \
+			--max-chain-depth=1 --split-factor=0 2>err &&
+		test_grep "split-factor must be greater than zero" err &&
+
+		test_must_fail git multi-pack-index compact \
+			--split-factor=2 \
+			"$(nth_line 1 "$midx_chain")" \
+			"$(nth_line 2 "$midx_chain")" 2>err &&
+		test_grep "cannot use --split-factor without --max-chain-depth" err
+	)
+'
+
 test_done