odb: add write_packfile, for_each_unique_abbrev, convert_object_id

Add three vtable methods to odb_source that were not part of the
recent ps/odb-sources and ps/object-counting series:

 - write_packfile: ingest a pack from a file descriptor. The files
   backend chooses between index-pack (large packs) and
   unpack-objects (small packs below fetch.unpackLimit). Options
   cover thin-pack fixing, promisor marking, fsck, lockfile
   capture, and shallow file passing.

 - for_each_unique_abbrev: iterate objects matching a hex prefix
   for disambiguation. Searches loose objects via oidtree, then
   multi-pack indices, then non-MIDX packs.

 - convert_object_id: translate between hash algorithms using the
   loose object map. Used during SHA-1 to SHA-256 migration.

Also add ODB_SOURCE_HELPER to the source type enum, preparing for
the helper backend in the next commit.

The write_packfile vtable method replaces the pattern where callers
spawn index-pack/unpack-objects directly. fast-import already uses
odb_write_packfile() and this allows non-files backends to handle
pack ingestion through their own mechanism.

Signed-off-by: Aaron Paterson <apaterson@pm.me>

Author: MayCXC

builtin/fast-import.c

@@ -518,31 +518,6 @@ static struct object_entry *insert_object(struct object_id *oid)
 	return e;
 }
 
-static void invalidate_pack_id(unsigned int id)
-{
-	unsigned long lu;
-	struct tag *t;
-	struct hashmap_iter iter;
-	struct object_entry *e;
-
-	hashmap_for_each_entry(&object_table, &iter, e, ent) {
-		if (e->pack_id == id)
-			e->pack_id = MAX_PACK_ID;
-	}
-
-	for (lu = 0; lu < branch_table_sz; lu++) {
-		struct branch *b;
-
-		for (b = branch_table[lu]; b; b = b->table_next_branch)
-			if (b->pack_id == id)
-				b->pack_id = MAX_PACK_ID;
-	}
-
-	for (t = first_tag; t; t = t->next_tag)
-		if (t->pack_id == id)
-			t->pack_id = MAX_PACK_ID;
-}
-
 static unsigned int hc_str(const char *s, size_t len)
 {
 	unsigned int r = 0;
@@ -786,86 +761,21 @@ static void start_packfile(void)
 	all_packs[pack_id] = p;
 }
 
-static const char *create_index(void)
-{
-	const char *tmpfile;
-	struct pack_idx_entry **idx, **c, **last;
-	struct object_entry *e;
-	struct object_entry_pool *o;
-
-	/* Build the table of object IDs. */
-	ALLOC_ARRAY(idx, object_count);
-	c = idx;
-	for (o = blocks; o; o = o->next_pool)
-		for (e = o->next_free; e-- != o->entries;)
-			if (pack_id == e->pack_id)
-				*c++ = &e->idx;
-	last = idx + object_count;
-	if (c != last)
-		die(_("internal consistency error creating the index"));
-
-	tmpfile = write_idx_file(the_repository, NULL, idx, object_count,
-				 &pack_idx_opts, pack_data->hash);
-	free(idx);
-	return tmpfile;
-}
-
-static char *keep_pack(const char *curr_index_name)
-{
-	static const char *keep_msg = "fast-import";
-	struct strbuf name = STRBUF_INIT;
-	int keep_fd;
-
-	odb_pack_name(pack_data->repo, &name, pack_data->hash, "keep");
-	keep_fd = safe_create_file_with_leading_directories(pack_data->repo,
-							    name.buf);
-	if (keep_fd < 0)
-		die_errno(_("cannot create keep file"));
-	write_or_die(keep_fd, keep_msg, strlen(keep_msg));
-	if (close(keep_fd))
-		die_errno(_("failed to write keep file"));
-
-	odb_pack_name(pack_data->repo, &name, pack_data->hash, "pack");
-	if (finalize_object_file(pack_data->repo, pack_data->pack_name, name.buf))
-		die(_("cannot store pack file"));
-
-	odb_pack_name(pack_data->repo, &name, pack_data->hash, "idx");
-	if (finalize_object_file(pack_data->repo, curr_index_name, name.buf))
-		die(_("cannot store index file"));
-	free((void *)curr_index_name);
-	return strbuf_detach(&name, NULL);
-}
-
 static void unkeep_all_packs(void)
 {
 	struct strbuf name = STRBUF_INIT;
 	int k;
 
 	for (k = 0; k < pack_id; k++) {
 		struct packed_git *p = all_packs[k];
+		if (!p)
+			continue;
 		odb_pack_name(p->repo, &name, p->hash, "keep");
 		unlink_or_warn(name.buf);
 	}
 	strbuf_release(&name);
 }
 
-static int loosen_small_pack(const struct packed_git *p)
-{
-	struct child_process unpack = CHILD_PROCESS_INIT;
-
-	if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
-		die_errno(_("failed seeking to start of '%s'"), p->pack_name);
-
-	unpack.in = p->pack_fd;
-	unpack.git_cmd = 1;
-	unpack.stdout_to_stderr = 1;
-	strvec_push(&unpack.args, "unpack-objects");
-	if (!show_stats)
-		strvec_push(&unpack.args, "-q");
-
-	return run_command(&unpack);
-}
-
 static void end_packfile(void)
 {
 	static int running;
@@ -876,10 +786,7 @@ static void end_packfile(void)
 	running = 1;
 	clear_delta_base_cache();
 	if (object_count) {
-		struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources);
-		struct packed_git *new_p;
 		struct object_id cur_pack_oid;
-		char *idx_name;
 		int i;
 		struct branch *b;
 		struct tag *t;
@@ -891,26 +798,25 @@ static void end_packfile(void)
 					 object_count, cur_pack_oid.hash,
 					 pack_size);
 
-		if (object_count <= unpack_limit) {
-			if (!loosen_small_pack(pack_data)) {
-				invalidate_pack_id(pack_id);
-				goto discard_pack;
-			}
-		}
+		if (lseek(pack_data->pack_fd, 0, SEEK_SET) < 0)
+			die_errno(_("failed seeking to start of '%s'"),
+				  pack_data->pack_name);
 
-		close(pack_data->pack_fd);
-		idx_name = keep_pack(create_index());
+		if (odb_write_packfile(the_repository->objects,
+				       pack_data->pack_fd, NULL))
+			die(_("failed to ingest pack"));
 
-		/* Register the packfile with core git's machinery. */
-		new_p = packfile_store_load_pack(files->packed, idx_name, 1);
-		if (!new_p)
-			die(_("core Git rejected index %s"), idx_name);
-		all_packs[pack_id] = new_p;
-		free(idx_name);
+		/*
+		 * Non-files backends do not register a pack on disk,
+		 * so NULL out the slot to prevent use-after-free in
+		 * gfi_unpack_entry.
+		 */
+		all_packs[pack_id] = NULL;
 
 		/* Print the boundary */
 		if (pack_edges) {
-			fprintf(pack_edges, "%s:", new_p->pack_name);
+			fprintf(pack_edges, "pack-%s:",
+				hash_to_hex(pack_data->hash));
 			for (i = 0; i < branch_table_sz; i++) {
 				for (b = branch_table[i]; b; b = b->table_next_branch) {
 					if (b->pack_id == pack_id)
@@ -930,7 +836,6 @@ static void end_packfile(void)
 		pack_id++;
 	}
 	else {
-discard_pack:
 		close(pack_data->pack_fd);
 		unlink_or_warn(pack_data->pack_name);
 	}
@@ -956,7 +861,6 @@ static int store_object(
 	struct object_id *oidout,
 	uintmax_t mark)
 {
-	struct odb_source *source;
 	void *out, *delta;
 	struct object_entry *e;
 	unsigned char hdr[96];
@@ -982,11 +886,7 @@ static int store_object(
 		return 1;
 	}
 
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-
-		if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
-			continue;
+	if (odb_has_object(the_repository->objects, &oid, 0)) {
 		e->type = type;
 		e->pack_id = MAX_PACK_ID;
 		e->idx.offset = 1; /* just not zero! */
@@ -1106,7 +1006,6 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
 	size_t in_sz = 64 * 1024, out_sz = 64 * 1024;
 	unsigned char *in_buf = xmalloc(in_sz);
 	unsigned char *out_buf = xmalloc(out_sz);
-	struct odb_source *source;
 	struct object_entry *e;
 	struct object_id oid;
 	unsigned long hdrlen;
@@ -1189,11 +1088,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
 		goto out;
 	}
 
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-
-		if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid))
-			continue;
+	if (odb_has_object(the_repository->objects, &oid, 0)) {
 		e->type = OBJ_BLOB;
 		e->pack_id = MAX_PACK_ID;
 		e->idx.offset = 1; /* just not zero! */
@@ -1239,6 +1134,16 @@ static void *gfi_unpack_entry(
 {
 	enum object_type type;
 	struct packed_git *p = all_packs[oe->pack_id];
+	if (!p) {
+		/*
+		 * Pack was ingested by a non-files backend via
+		 * odb_write_packfile() and is no longer on disk.
+		 * Read the object back through the ODB instead.
+		 */
+		enum object_type odb_type;
+		return odb_read_object(the_repository->objects,
+				       &oe->idx.oid, &odb_type, sizep);
+	}
 	if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) {
 		/* The object is stored in the packfile we are writing to
 		 * and we have modified it since the last time we scanned

builtin/pack-objects.c

@@ -1541,57 +1541,15 @@ static int have_duplicate_entry(const struct object_id *oid,
 
 static int want_cruft_object_mtime(struct repository *r,
 				   const struct object_id *oid,
-				   unsigned flags, uint32_t mtime)
+				   unsigned flags UNUSED, uint32_t mtime UNUSED)
 {
-	struct odb_source *source;
-
-	for (source = r->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-		struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags);
-
-		for (; *cache; cache++) {
-			struct packed_git *p = *cache;
-			off_t ofs;
-			uint32_t candidate_mtime;
-
-			ofs = find_pack_entry_one(oid, p);
-			if (!ofs)
-				continue;
-
-			/*
-			 * We have a copy of the object 'oid' in a non-cruft
-			 * pack. We can avoid packing an additional copy
-			 * regardless of what the existing copy's mtime is since
-			 * it is outside of a cruft pack.
-			 */
-			if (!p->is_cruft)
-				return 0;
-
-			/*
-			 * If we have a copy of the object 'oid' in a cruft
-			 * pack, then either read the cruft pack's mtime for
-			 * that object, or, if that can't be loaded, assume the
-			 * pack's mtime itself.
-			 */
-			if (!load_pack_mtimes(p)) {
-				uint32_t pos;
-				if (offset_to_pack_pos(p, ofs, &pos) < 0)
-					continue;
-				candidate_mtime = nth_packed_mtime(p, pos);
-			} else {
-				candidate_mtime = p->mtime;
-			}
-
-			/*
-			 * We have a surviving copy of the object in a cruft
-			 * pack whose mtime is greater than or equal to the one
-			 * we are considering. We can thus avoid packing an
-			 * additional copy of that object.
-			 */
-			if (mtime <= candidate_mtime)
-				return 0;
-		}
-	}
+	/*
+	 * Check if the object exists in a kept source. Dispatches through
+	 * the vtable: files backends check kept packs, non-files backends
+	 * check their own kept tracking via OBJECT_INFO_KEPT_ONLY.
+	 */
+	if (odb_has_object_kept(r->objects, oid))
+		return 0;
 
 	return -1;
 }
@@ -1657,7 +1615,7 @@ static int want_found_object(const struct object_id *oid, int exclude,
 				return 0;
 			if (ignore_packed_keep_in_core && p->pack_keep_in_core)
 				return 0;
-			if (has_object_kept_pack(p->repo, oid, flags))
+			if (odb_has_object_kept(p->repo->objects, oid))
 				return 0;
 		} else {
 			/*
@@ -1726,8 +1684,6 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 				     uint32_t found_mtime)
 {
 	int want;
-	struct packfile_list_entry *e;
-	struct odb_source *source;
 
 	if (!exclude && local) {
 		/*
@@ -1757,25 +1713,18 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 
 	odb_prepare_alternates(the_repository->objects);
 
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct multi_pack_index *m = get_multi_pack_index(source);
-		struct pack_entry e;
-
-		if (m && fill_midx_entry(m, oid, &e)) {
-			want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime);
-			if (want != -1)
-				return want;
-		}
-	}
-
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
+	{
+		struct object_info oi = OBJECT_INFO_INIT;
 
-		for (e = files->packed->packs.head; e; e = e->next) {
-			struct packed_git *p = e->pack;
-			want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
-			if (!exclude && want > 0)
-				packfile_list_prepend(&files->packed->packs, p);
+		if (!odb_read_object_info_extended(the_repository->objects,
+						    oid, &oi,
+						    OBJECT_INFO_QUICK) &&
+		    oi.whence == OI_PACKED) {
+			struct packed_git *p = oi.u.packed.pack;
+			want = want_object_in_pack_one(p, oid, exclude,
+						       found_pack,
+						       found_offset,
+						       found_mtime);
 			if (want != -1)
 				return want;
 		}
@@ -4065,7 +4014,7 @@ static void show_cruft_commit(struct commit *commit, void *data)
 
 static int cruft_include_check_obj(struct object *obj, void *data UNUSED)
 {
-	return !has_object_kept_pack(to_pack.repo, &obj->oid, KEPT_PACK_IN_CORE);
+	return !odb_has_object_kept(to_pack.repo->objects, &obj->oid);
 }
 
 static int cruft_include_check(struct commit *commit, void *data)
@@ -4365,17 +4314,15 @@ static void add_objects_in_unpacked_packs(void)
 
 	odb_prepare_alternates(to_pack.repo->objects);
 	for (source = to_pack.repo->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-
 		if (!source->local)
 			continue;
 
-		if (packfile_store_for_each_object(files->packed, &oi,
-						   add_object_in_unpacked_pack, NULL,
-						   ODB_FOR_EACH_OBJECT_PACK_ORDER |
-						   ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
-						   ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
-						   ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
+		if (odb_source_for_each_object(source, &oi,
+					       add_object_in_unpacked_pack, NULL,
+					       ODB_FOR_EACH_OBJECT_PACK_ORDER |
+					       ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
+					       ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
+					       ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
 			die(_("cannot open pack index"));
 	}
 }