odb: add write_packfile, for_each_unique_abbrev, convert_object_id

Add three vtable methods to odb_source that were not part of the
recent ps/odb-sources and ps/object-counting series:

 - write_packfile: ingest a pack from a file descriptor. The files
   backend chooses between index-pack (large packs) and
   unpack-objects (small packs below fetch.unpackLimit). Options
   cover thin-pack fixing, promisor marking, fsck, lockfile
   capture, and shallow file passing.

 - for_each_unique_abbrev: iterate objects matching a hex prefix
   for disambiguation. Searches loose objects via oidtree, then
   multi-pack indices, then non-MIDX packs.

 - convert_object_id: translate between hash algorithms using the
   loose object map. Used during SHA-1 to SHA-256 migration.

Also add ODB_SOURCE_HELPER to the source type enum, preparing for
the helper backend in the next commit.

The write_packfile vtable method replaces the pattern where callers
spawn index-pack/unpack-objects directly. fast-import already uses
odb_write_packfile() and this allows non-files backends to handle
pack ingestion through their own mechanism.

Signed-off-by: Aaron Paterson <apaterson@pm.me>

Author: MayCXC

builtin/cat-file.c

@@ -24,6 +24,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "odb.h"
+#include "odb/source.h"
 #include "odb/streaming.h"
 #include "replace-object.h"
 #include "promisor-remote.h"
@@ -859,8 +860,17 @@ static void batch_each_object(struct batch_options *opt,
 	 */
 	odb_prepare_alternates(the_repository->objects);
 	for (source = the_repository->objects->sources; source; source = source->next) {
-		int ret = odb_source_loose_for_each_object(source, NULL, batch_one_object_oi,
-							   &payload, flags);
+		int ret;
+		if (!source->packed) {
+			/*
+			 * Non-files source: dispatch through vtable.
+			 */
+			ret = odb_source_for_each_object(source, NULL,
+				batch_one_object_oi, &payload, flags);
+		} else {
+			ret = odb_source_loose_for_each_object(source, NULL,
+				batch_one_object_oi, &payload, flags);
+		}
 		if (ret)
 			break;
 	}
@@ -882,11 +892,14 @@ static void batch_each_object(struct batch_options *opt,
 		struct object_info oi = { 0 };
 
 		for (source = the_repository->objects->sources; source; source = source->next) {
-			struct odb_source_files *files = odb_source_files_downcast(source);
-			int ret = packfile_store_for_each_object(files->packed, &oi,
-								 batch_one_object_oi, &payload, flags);
-			if (ret)
-				break;
+			if (!source->packed)
+				continue;
+			{
+				int ret = packfile_store_for_each_object(source->packed, &oi,
+									 batch_one_object_oi, &payload, flags);
+				if (ret)
+					break;
+			}
 		}
 	}
 

builtin/fsck.c

@@ -17,6 +17,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "odb.h"
+#include "odb/source.h"
 #include "odb/streaming.h"
 #include "path.h"
 #include "read-cache-ll.h"
@@ -769,24 +770,56 @@ static int fsck_subdir(unsigned int nr, const char *path UNUSED, void *data)
 	return 0;
 }
 
+static int fsck_vtable_object(const struct object_id *oid,
+			      struct object_info *oi UNUSED,
+			      void *data UNUSED)
+{
+	struct object *obj;
+
+	obj = parse_object(the_repository, oid);
+	if (!obj) {
+		errors_found |= ERROR_OBJECT;
+		error(_("%s: object missing"),
+		      describe_object(oid));
+		return 0;
+	}
+	obj->flags |= HAS_OBJ;
+	if (fsck_obj(obj, NULL, 0))
+		errors_found |= ERROR_OBJECT;
+	return 0;
+}
+
 static void fsck_source(struct odb_source *source)
 {
 	struct progress *progress = NULL;
-	struct for_each_loose_cb cb_data = {
-		.progress = progress,
-	};
 
 	if (verbose)
 		fprintf_ln(stderr, _("Checking object directory"));
 
-	if (show_progress)
-		progress = start_progress(the_repository,
-					  _("Checking object directories"), 256);
+	if (!source->packed) {
+		/*
+		 * Non-files source: iterate objects through the vtable
+		 * and fsck each one.
+		 */
+		odb_source_for_each_object(source, NULL,
+					   fsck_vtable_object, NULL, 0);
+		return;
+	}
 
-	for_each_loose_file_in_source(source, fsck_loose,
-				      fsck_cruft, fsck_subdir, &cb_data);
-	display_progress(progress, 256);
-	stop_progress(&progress);
+	{
+		struct for_each_loose_cb cb_data = {
+			.progress = progress,
+		};
+
+		if (show_progress)
+			progress = start_progress(the_repository,
+						  _("Checking object directories"), 256);
+
+		for_each_loose_file_in_source(source, fsck_loose,
+					      fsck_cruft, fsck_subdir, &cb_data);
+		display_progress(progress, 256);
+		stop_progress(&progress);
+	}
 }
 
 static int fsck_cache_tree(struct cache_tree *it, const char *index_path)

builtin/pack-objects.c

@@ -32,6 +32,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "odb.h"
+#include "odb/source.h"
 #include "odb/streaming.h"
 #include "replace-object.h"
 #include "dir.h"
@@ -1541,57 +1542,15 @@ static int have_duplicate_entry(const struct object_id *oid,
 
 static int want_cruft_object_mtime(struct repository *r,
 				   const struct object_id *oid,
-				   unsigned flags, uint32_t mtime)
+				   unsigned flags UNUSED, uint32_t mtime UNUSED)
 {
-	struct odb_source *source;
-
-	for (source = r->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-		struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags);
-
-		for (; *cache; cache++) {
-			struct packed_git *p = *cache;
-			off_t ofs;
-			uint32_t candidate_mtime;
-
-			ofs = find_pack_entry_one(oid, p);
-			if (!ofs)
-				continue;
-
-			/*
-			 * We have a copy of the object 'oid' in a non-cruft
-			 * pack. We can avoid packing an additional copy
-			 * regardless of what the existing copy's mtime is since
-			 * it is outside of a cruft pack.
-			 */
-			if (!p->is_cruft)
-				return 0;
-
-			/*
-			 * If we have a copy of the object 'oid' in a cruft
-			 * pack, then either read the cruft pack's mtime for
-			 * that object, or, if that can't be loaded, assume the
-			 * pack's mtime itself.
-			 */
-			if (!load_pack_mtimes(p)) {
-				uint32_t pos;
-				if (offset_to_pack_pos(p, ofs, &pos) < 0)
-					continue;
-				candidate_mtime = nth_packed_mtime(p, pos);
-			} else {
-				candidate_mtime = p->mtime;
-			}
-
-			/*
-			 * We have a surviving copy of the object in a cruft
-			 * pack whose mtime is greater than or equal to the one
-			 * we are considering. We can thus avoid packing an
-			 * additional copy of that object.
-			 */
-			if (mtime <= candidate_mtime)
-				return 0;
-		}
-	}
+	/*
+	 * Check if the object exists in a kept source. Dispatches through
+	 * the vtable: files backends check kept packs, non-files backends
+	 * check their own kept tracking via OBJECT_INFO_KEPT_ONLY.
+	 */
+	if (odb_has_object_kept(r->objects, oid))
+		return 0;
 
 	return -1;
 }
@@ -1657,7 +1616,7 @@ static int want_found_object(const struct object_id *oid, int exclude,
 				return 0;
 			if (ignore_packed_keep_in_core && p->pack_keep_in_core)
 				return 0;
-			if (has_object_kept_pack(p->repo, oid, flags))
+			if (odb_has_object_kept(p->repo->objects, oid))
 				return 0;
 		} else {
 			/*
@@ -1726,8 +1685,6 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 				     uint32_t found_mtime)
 {
 	int want;
-	struct packfile_list_entry *e;
-	struct odb_source *source;
 
 	if (!exclude && local) {
 		/*
@@ -1757,25 +1714,18 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 
 	odb_prepare_alternates(the_repository->objects);
 
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct multi_pack_index *m = get_multi_pack_index(source);
-		struct pack_entry e;
-
-		if (m && fill_midx_entry(m, oid, &e)) {
-			want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime);
-			if (want != -1)
-				return want;
-		}
-	}
-
-	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
+	{
+		struct object_info oi = OBJECT_INFO_INIT;
 
-		for (e = files->packed->packs.head; e; e = e->next) {
-			struct packed_git *p = e->pack;
-			want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
-			if (!exclude && want > 0)
-				packfile_list_prepend(&files->packed->packs, p);
+		if (!odb_read_object_info_extended(the_repository->objects,
+						    oid, &oi,
+						    OBJECT_INFO_QUICK) &&
+		    oi.whence == OI_PACKED) {
+			struct packed_git *p = oi.u.packed.pack;
+			want = want_object_in_pack_one(p, oid, exclude,
+						       found_pack,
+						       found_offset,
+						       found_mtime);
 			if (want != -1)
 				return want;
 		}
@@ -4065,7 +4015,7 @@ static void show_cruft_commit(struct commit *commit, void *data)
 
 static int cruft_include_check_obj(struct object *obj, void *data UNUSED)
 {
-	return !has_object_kept_pack(to_pack.repo, &obj->oid, KEPT_PACK_IN_CORE);
+	return !odb_has_object_kept(to_pack.repo->objects, &obj->oid);
 }
 
 static int cruft_include_check(struct commit *commit, void *data)
@@ -4365,17 +4315,15 @@ static void add_objects_in_unpacked_packs(void)
 
 	odb_prepare_alternates(to_pack.repo->objects);
 	for (source = to_pack.repo->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-
 		if (!source->local)
 			continue;
 
-		if (packfile_store_for_each_object(files->packed, &oi,
-						   add_object_in_unpacked_pack, NULL,
-						   ODB_FOR_EACH_OBJECT_PACK_ORDER |
-						   ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
-						   ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
-						   ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
+		if (odb_source_for_each_object(source, &oi,
+					       add_object_in_unpacked_pack, NULL,
+					       ODB_FOR_EACH_OBJECT_PACK_ORDER |
+					       ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
+					       ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
+					       ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
 			die(_("cannot open pack index"));
 	}
 }

loose.c

@@ -128,13 +128,17 @@ int repo_read_loose_object_map(struct repository *repo)
 
 int repo_write_loose_object_map(struct repository *repo)
 {
-	struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources);
-	kh_oid_map_t *map = files->loose->map->to_compat;
+	struct odb_source *source = repo->objects->sources;
+	kh_oid_map_t *map;
 	struct lock_file lock;
 	int fd;
 	khiter_t iter;
 	struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
 
+	if (!source->loose || !source->loose->map)
+		return 0;
+	map = source->loose->map->to_compat;
+
 	if (!should_use_loose_object_map(repo))
 		return 0;
 
@@ -235,8 +239,10 @@ int repo_loose_object_map_oid(struct repository *repo,
 	khiter_t pos;
 
 	for (source = repo->objects->sources; source; source = source->next) {
-		struct odb_source_files *files = odb_source_files_downcast(source);
-		struct loose_object_map *loose_map = files->loose->map;
+		struct loose_object_map *loose_map;
+		if (!source->loose)
+			continue;
+		loose_map = source->loose->map;
 		if (!loose_map)
 			continue;
 		map = (to == repo->compat_hash_algo) ?

midx.c

@@ -101,9 +101,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
 
 struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
 {
-	struct odb_source_files *files = odb_source_files_downcast(source);
-	packfile_store_prepare(files->packed);
-	return files->packed->midx;
+	if (!source->packed)
+		return NULL;
+	packfile_store_prepare(source->packed);
+	return source->packed->midx;
 }
 
 static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,