test-tool: add a helper to synthesize large packfiles

To test Git's behavior with very large pack files, we need a way to
generate such files quickly.

A naive approach using only readily-available Git commands would take
over 10 hours for a 4GB pack file, which is prohibitive.

Side-stepping Git's machinery and actual zlib compression by writing
uncompressed content with the appropriate zlib header makes things
much faster. The fastest method using this approach generates many
small, unreachable blob objects and takes about 1.5 minutes for 4GB.
However, this cannot be used because we need to test git clone, which
requires a reachable commit history.

Generating many reachable commits with small, uncompressed blobs takes
about 4 minutes for 4GB. But this approach 1) does not reproduce the
issues we want to fix (which require individual objects larger than
4GB) and 2) is comparatively slow because of the many SHA-1
calculations.

The approach taken here generates a single large blob (filled with NUL
bytes), along with the trees and commits needed to make it reachable.
This takes about 2.5 minutes for 4.5GB, which is the fastest option
that produces a valid, clonable repository with an object large enough
to trigger the bugs we want to test.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Author: dscho

Makefile

@@ -875,6 +875,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o
 TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o
 TEST_BUILTINS_OBJS += test-submodule.o
 TEST_BUILTINS_OBJS += test-subprocess.o
+TEST_BUILTINS_OBJS += test-synthesize.o
 TEST_BUILTINS_OBJS += test-trace2.o
 TEST_BUILTINS_OBJS += test-truncate.o
 TEST_BUILTINS_OBJS += test-userdiff.o

t/helper/meson.build

@@ -70,6 +70,7 @@ test_tool_sources = [
   'test-submodule-nested-repo-config.c',
   'test-submodule.c',
   'test-subprocess.c',
+  'test-synthesize.c',
   'test-tool.c',
   'test-trace2.c',
   'test-truncate.c',

t/helper/test-synthesize.c

@@ -0,0 +1,295 @@
+#include "test-tool.h"
+#include "git-compat-util.h"
+#include "hex.h"
+#include "object.h"
+#include "strbuf.h"
+#include "parse-options.h"
+#include "hash.h"
+#include "git-zlib.h"
+
+/*
+ * Write a pack object header for the given type and size.
+ * Returns the number of bytes written to the buffer.
+ */
+static size_t write_pack_object_header(unsigned char *buf, enum object_type type, size_t size)
+{
+	unsigned char *p = buf;
+	*p = (type << 4) | (size & 0xf);
+	size >>= 4;
+	while (size) {
+		*p++ |= 0x80;
+		*p = size & 0x7f;
+		size >>= 7;
+	}
+	p++;
+	return p - buf;
+}
+
+/*
+ * Write data as an uncompressed zlib stream.
+ * For data larger than 64KB, writes multiple uncompressed blocks.
+ * If data is NULL, writes zeros.
+ * Updates the pack checksum context.
+ */
+static void write_uncompressed_zlib(FILE *f, struct git_hash_ctx *pack_ctx,
+				    const void *data, size_t len,
+				    const struct git_hash_algo *algo)
+{
+	unsigned char zlib_header[2] = { 0x78, 0x01 }; /* CMF, FLG */
+	unsigned char block_header[5];
+	static unsigned char zeros[0xffff];
+	const unsigned char *p = data;
+	size_t remaining = len;
+	uint32_t adler = 1L; /* adler32 initial value */
+	unsigned char adler_buf[4];
+
+	/* Write zlib header */
+	fwrite(zlib_header, 1, 2, f);
+	algo->update_fn(pack_ctx, zlib_header, 2);
+
+	/* Write uncompressed blocks (max 64KB each) */
+	do {
+		size_t block_len = remaining > 0xffff ? 0xffff : remaining;
+		int is_final = (block_len == remaining);
+		const unsigned char *block_data = data ? p : zeros;
+
+		block_header[0] = is_final ? 0x01 : 0x00;
+		block_header[1] = block_len & 0xff;
+		block_header[2] = (block_len >> 8) & 0xff;
+		block_header[3] = block_header[1] ^ 0xff;
+		block_header[4] = block_header[2] ^ 0xff;
+
+		fwrite(block_header, 1, 5, f);
+		algo->update_fn(pack_ctx, block_header, 5);
+
+		if (block_len) {
+			fwrite(block_data, 1, block_len, f);
+			algo->update_fn(pack_ctx, block_data, block_len);
+			adler = adler32(adler, block_data, block_len);
+		}
+
+		if (data)
+			p += block_len;
+		remaining -= block_len;
+	} while (remaining > 0);
+
+	/* Write adler32 checksum */
+	put_be32(adler_buf, adler);
+	fwrite(adler_buf, 1, 4, f);
+	algo->update_fn(pack_ctx, adler_buf, 4);
+}
+
+/*
+ * Write an uncompressed object to the pack file.
+ * Updates the pack checksum context.
+ */
+static void write_pack_object(FILE *f, struct git_hash_ctx *pack_ctx,
+			      enum object_type type, const void *data, size_t len,
+			      const struct git_hash_algo *algo)
+{
+	unsigned char header[32];
+	size_t header_len;
+
+	/* Write pack object header */
+	header_len = write_pack_object_header(header, type, len);
+	fwrite(header, 1, header_len, f);
+	algo->update_fn(pack_ctx, header, header_len);
+
+	/* Write the data as uncompressed zlib */
+	write_uncompressed_zlib(f, pack_ctx, data, len, algo);
+}
+
+/*
+ * Compute the object ID for a given object.
+ */
+static void hash_object(struct object_id *oid, enum object_type type,
+			const void *data, size_t len,
+			const struct git_hash_algo *algo)
+{
+	struct git_hash_ctx ctx;
+	char header[32];
+	int header_len;
+
+	header_len = xsnprintf(header, sizeof(header), "%s %"PRIuMAX,
+			       type_name(type), (uintmax_t)len) + 1;
+
+	algo->init_fn(&ctx);
+	algo->update_fn(&ctx, header, header_len);
+	algo->update_fn(&ctx, data, len);
+	algo->final_oid_fn(oid, &ctx);
+}
+
+/*
+ * Compute the object ID for a large object filled with zeros.
+ */
+static void hash_large_zero_object(struct object_id *oid, enum object_type type,
+				   size_t len, const struct git_hash_algo *algo)
+{
+	struct git_hash_ctx ctx;
+	char header[32];
+	int header_len;
+	static unsigned char zeros[0xffff];
+	size_t remaining;
+
+	header_len = xsnprintf(header, sizeof(header), "%s %"PRIuMAX,
+			       type_name(type), (uintmax_t)len) + 1;
+
+	algo->init_fn(&ctx);
+	algo->update_fn(&ctx, header, header_len);
+	for (remaining = len; remaining; ) {
+		size_t chunk = remaining > sizeof(zeros) ? sizeof(zeros) : remaining;
+		algo->update_fn(&ctx, zeros, chunk);
+		remaining -= chunk;
+	}
+	algo->final_oid_fn(oid, &ctx);
+}
+
+/*
+ * Write a large object (zeros) to the pack file, streaming to avoid
+ * allocating the entire object in memory.
+ */
+static void write_large_pack_object(FILE *f, struct git_hash_ctx *pack_ctx,
+				    enum object_type type, size_t len,
+				    struct object_id *oid,
+				    const struct git_hash_algo *algo)
+{
+	unsigned char header[32];
+	size_t header_len;
+
+	/* Compute the object ID */
+	hash_large_zero_object(oid, type, len, algo);
+
+	/* Write pack object header */
+	header_len = write_pack_object_header(header, type, len);
+	fwrite(header, 1, header_len, f);
+	algo->update_fn(pack_ctx, header, header_len);
+
+	/* Write the zeros as uncompressed zlib (NULL means zeros) */
+	write_uncompressed_zlib(f, pack_ctx, NULL, len, algo);
+}
+
+/*
+ * Generate a pack file with a single large (>4GB) reachable object.
+ *
+ * Creates:
+ *   1. A large blob (all NUL bytes)
+ *   2. A tree containing that blob as "file"
+ *   3. A commit using that tree
+ *   4. The empty tree
+ *   5. A child commit using the empty tree
+ *
+ * This is useful for testing that Git can handle objects larger than 4GB.
+ */
+static int generate_pack_with_large_object(const char *path, size_t blob_size,
+					   const struct git_hash_algo *algo)
+{
+	FILE *f = fopen_for_writing(path);
+	struct git_hash_ctx pack_ctx;
+	char header[1024];
+	struct object_id blob_oid, tree_oid, commit_oid, empty_tree_oid, final_commit_oid;
+	struct strbuf buf = STRBUF_INIT;
+	size_t object_count = 5; /* large blob, tree, commit, empty tree, final commit */
+
+	algo->init_fn(&pack_ctx);
+
+	/* Write pack header */
+	memcpy(header, "PACK", 4);
+	put_be32(header + 4, 2);
+	put_be32(header + 8, object_count);
+	fwrite(header, 1, 12, f);
+	algo->update_fn(&pack_ctx, header, 12);
+
+	/* 1. Write the large blob */
+	write_large_pack_object(f, &pack_ctx, OBJ_BLOB, blob_size, &blob_oid, algo);
+
+	/* 2. Write tree containing the blob as "file" */
+	strbuf_addf(&buf, "100644 file%c", '\0');
+	strbuf_add(&buf, blob_oid.hash, algo->rawsz);
+	hash_object(&tree_oid, OBJ_TREE, buf.buf, buf.len, algo);
+	write_pack_object(f, &pack_ctx, OBJ_TREE, buf.buf, buf.len, algo);
+
+	/* 3. Write commit using that tree */
+	strbuf_reset(&buf);
+	strbuf_addf(&buf,
+		    "tree %s\n"
+		    "author A U Thor <author@example.com> 1234567890 +0000\n"
+		    "committer C O Mitter <committer@example.com> 1234567890 +0000\n"
+		    "\n"
+		    "Large blob commit\n",
+		    oid_to_hex(&tree_oid));
+	hash_object(&commit_oid, OBJ_COMMIT, buf.buf, buf.len, algo);
+	write_pack_object(f, &pack_ctx, OBJ_COMMIT, buf.buf, buf.len, algo);
+
+	/* 4. Write the empty tree */
+	hash_object(&empty_tree_oid, OBJ_TREE, "", 0, algo);
+	write_pack_object(f, &pack_ctx, OBJ_TREE, "", 0, algo);
+
+	/* 5. Write final commit using empty tree, with previous commit as parent */
+	strbuf_reset(&buf);
+	strbuf_addf(&buf,
+		    "tree %s\n"
+		    "parent %s\n"
+		    "author A U Thor <author@example.com> 1234567890 +0000\n"
+		    "committer C O Mitter <committer@example.com> 1234567890 +0000\n"
+		    "\n"
+		    "Empty tree commit\n",
+		    oid_to_hex(&empty_tree_oid),
+		    oid_to_hex(&commit_oid));
+	hash_object(&final_commit_oid, OBJ_COMMIT, buf.buf, buf.len, algo);
+	write_pack_object(f, &pack_ctx, OBJ_COMMIT, buf.buf, buf.len, algo);
+
+	/* Write pack trailer (checksum) */
+	algo->final_fn((unsigned char *)header, &pack_ctx);
+	fwrite(header, 1, algo->rawsz, f);
+
+	fclose(f);
+
+	strbuf_release(&buf);
+
+	/* Print the final commit OID so caller can set up refs */
+	printf("%s\n", oid_to_hex(&final_commit_oid));
+
+	return 0;
+}
+
+static int cmd__synthesize__pack(int argc, const char **argv,
+				 const char *prefix UNUSED,
+				 struct repository *repo UNUSED)
+{
+	const struct git_hash_algo *algo = hash_algos + GIT_HASH_SHA1;
+	size_t count;
+	const char *path;
+	const char * const usage[] = {
+		"test-tool synthesize pack <count> <filename>",
+		NULL
+	};
+	struct option options[] = {
+		OPT_END()
+	};
+
+	argc = parse_options(argc, argv, NULL, options, usage,
+			     PARSE_OPT_KEEP_ARGV0);
+	if (argc != 3)
+		usage_with_options(usage, options);
+
+	count = strtoumax(argv[1], NULL, 10);
+	path = argv[2];
+
+	return !!generate_pack_with_large_object(path, count, algo);
+}
+
+int cmd__synthesize(int argc, const char **argv)
+{
+	const char *prefix = NULL;
+	char const * const synthesize_usage[] = {
+		"test-tool synthesize pack <options>",
+		NULL,
+	};
+	parse_opt_subcommand_fn *fn = NULL;
+	struct option options[] = {
+		OPT_SUBCOMMAND("pack", &fn, cmd__synthesize__pack),
+		OPT_END()
+	};
+	argc = parse_options(argc, argv, prefix, options, synthesize_usage, 0);
+	return !!fn(argc, argv, prefix, NULL);
+}

t/helper/test-tool.c

@@ -83,6 +83,7 @@ static struct test_cmd cmds[] = {
 	{ "submodule-config", cmd__submodule_config },
 	{ "submodule-nested-repo-config", cmd__submodule_nested_repo_config },
 	{ "subprocess", cmd__subprocess },
+	{ "synthesize", cmd__synthesize },
 	{ "trace2", cmd__trace2 },
 	{ "truncate", cmd__truncate },
 	{ "userdiff", cmd__userdiff },

t/helper/test-tool.h

@@ -76,6 +76,7 @@ int cmd__submodule(int argc, const char **argv);
 int cmd__submodule_config(int argc, const char **argv);
 int cmd__submodule_nested_repo_config(int argc, const char **argv);
 int cmd__subprocess(int argc, const char **argv);
+int cmd__synthesize(int argc, const char **argv);
 int cmd__trace2(int argc, const char **argv);
 int cmd__truncate(int argc, const char **argv);
 int cmd__userdiff(int argc, const char **argv);