Merge branch 'ps/odb-source-packed' into ps/odb-generalize-prepare

* ps/odb-source-packed:
  odb/source-packed: drop pointer to "files" parent source
  midx: refactor interfaces to work on "packed" source
  odb/source-packed: stub out remaining functions
  odb/source-packed: wire up `freshen_object()` callback
  odb/source-packed: wire up `find_abbrev_len()` callback
  odb/source-packed: wire up `count_objects()` callback
  odb/source-packed: wire up `for_each_object()` callback
  odb/source-packed: wire up `read_object_stream()` callback
  odb/source-packed: wire up `read_object_info()` callback
  packfile: use higher-level interface to implement `has_object_pack()`
  odb/source-packed: wire up `reprepare()` callback
  odb/source-packed: wire up `close()` callback
  odb/source-packed: start converting to a proper `struct odb_source`
  odb/source-packed: store pointer to "files" instead of generic source
  packfile: move packed source into "odb/" subsystem
  packfile: split out packfile list logic
  packfile: rename `struct packfile_store` to `odb_source_packed`

Author: gitster

Makefile

@@ -1218,6 +1218,7 @@ LIB_OBJS += odb/source.o
 LIB_OBJS += odb/source-files.o
 LIB_OBJS += odb/source-inmemory.o
 LIB_OBJS += odb/source-loose.o
+LIB_OBJS += odb/source-packed.o
 LIB_OBJS += odb/streaming.o
 LIB_OBJS += odb/transaction.o
 LIB_OBJS += oid-array.o
@@ -1233,6 +1234,7 @@ LIB_OBJS += pack-refs.o
 LIB_OBJS += pack-revindex.o
 LIB_OBJS += pack-write.o
 LIB_OBJS += packfile.o
+LIB_OBJS += packfile-list.o
 LIB_OBJS += pager.o
 LIB_OBJS += parallel-checkout.o
 LIB_OBJS += parse.o

builtin/cat-file.c

@@ -915,8 +915,8 @@ static void batch_each_object(struct batch_options *opt,
 
 		for (source = the_repository->objects->sources; source; source = source->next) {
 			struct odb_source_files *files = odb_source_files_downcast(source);
-			int ret = packfile_store_for_each_object(files->packed, &oi,
-								 batch_one_object_oi, &payload, &opts);
+			int ret = odb_source_for_each_object(&files->packed->base, &oi,
+							     batch_one_object_oi, &payload, &opts);
 			if (ret)
 				break;
 		}

builtin/grep.c

@@ -1363,7 +1363,7 @@ int cmd_grep(int argc,
 			odb_prepare_alternates(the_repository->objects);
 			for (source = the_repository->objects->sources; source; source = source->next) {
 				struct odb_source_files *files = odb_source_files_downcast(source);
-				packfile_store_prepare(files->packed);
+				odb_source_packed_prepare(files->packed);
 			}
 		}
 

builtin/multi-pack-index.c

@@ -10,6 +10,7 @@
 #include "trace2.h"
 #include "odb.h"
 #include "odb/source.h"
+#include "odb/source-files.h"
 #include "replace-object.h"
 #include "repository.h"
 
@@ -85,12 +86,12 @@ static int parse_object_dir(const struct option *opt, const char *arg,
 	return 0;
 }
 
-static struct odb_source *handle_object_dir_option(struct repository *repo)
+static struct odb_source_files *handle_object_dir_option(struct repository *repo)
 {
 	struct odb_source *source = odb_find_source(repo->objects, opts.object_dir);
 	if (!source)
 		source = odb_add_to_alternates_memory(repo->objects, opts.object_dir);
-	return source;
+	return odb_source_files_downcast(source);
 }
 
 static struct option common_opts[] = {
@@ -167,7 +168,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
 			     N_("refs snapshot for selecting bitmap commits")),
 		OPT_END(),
 	};
-	struct odb_source *source;
+	struct odb_source_files *source;
 	int ret;
 
 	opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
@@ -211,7 +212,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
 
 		read_packs_from_stdin(&packs);
 
-		ret = write_midx_file_only(source, &packs,
+		ret = write_midx_file_only(source->packed, &packs,
 					   opts.preferred_pack,
 					   opts.refs_snapshot,
 					   opts.incremental_base, opts.flags);
@@ -223,7 +224,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
 
 	}
 
-	ret = write_midx_file(source, opts.preferred_pack,
+	ret = write_midx_file(source->packed, opts.preferred_pack,
 			      opts.refs_snapshot, opts.flags);
 
 	free(opts.refs_snapshot);
@@ -237,7 +238,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 	struct multi_pack_index *m, *cur;
 	struct multi_pack_index *from_midx = NULL;
 	struct multi_pack_index *to_midx = NULL;
-	struct odb_source *source;
+	struct odb_source_files *source;
 	int ret;
 
 	struct option *options;
@@ -282,7 +283,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 
 	FREE_AND_NULL(options);
 
-	m = get_multi_pack_index(source);
+	m = get_multi_pack_index(source->packed);
 
 	for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) {
 		const char *midx_csum = midx_get_checksum_hex(cur);
@@ -305,7 +306,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 			die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]);
 	}
 
-	ret = write_midx_file_compact(source, from_midx, to_midx,
+	ret = write_midx_file_compact(source->packed, from_midx, to_midx,
 				      opts.incremental_base, opts.flags);
 
 	return ret;
@@ -319,7 +320,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv,
 	static struct option builtin_multi_pack_index_verify_options[] = {
 		OPT_END(),
 	};
-	struct odb_source *source;
+	struct odb_source_files *source;
 
 	options = add_common_options(builtin_multi_pack_index_verify_options);
 
@@ -337,7 +338,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv,
 
 	FREE_AND_NULL(options);
 
-	return verify_midx_file(source, opts.flags);
+	return verify_midx_file(source->packed, opts.flags);
 }
 
 static int cmd_multi_pack_index_expire(int argc, const char **argv,
@@ -348,7 +349,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv,
 	static struct option builtin_multi_pack_index_expire_options[] = {
 		OPT_END(),
 	};
-	struct odb_source *source;
+	struct odb_source_files *source;
 
 	options = add_common_options(builtin_multi_pack_index_expire_options);
 
@@ -366,7 +367,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv,
 
 	FREE_AND_NULL(options);
 
-	return expire_midx_packs(source, opts.flags);
+	return expire_midx_packs(source->packed, opts.flags);
 }
 
 static int cmd_multi_pack_index_repack(int argc, const char **argv,
@@ -379,7 +380,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv,
 		  N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")),
 		OPT_END(),
 	};
-	struct odb_source *source;
+	struct odb_source_files *source;
 
 	options = add_common_options(builtin_multi_pack_index_repack_options);
 
@@ -398,7 +399,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv,
 
 	FREE_AND_NULL(options);
 
-	return midx_repack(source, (size_t)opts.batch_size, opts.flags);
+	return midx_repack(source->packed, (size_t)opts.batch_size, opts.flags);
 }
 
 int cmd_multi_pack_index(int argc,

builtin/pack-objects.c

@@ -1777,7 +1777,8 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
 	odb_prepare_alternates(the_repository->objects);
 
 	for (source = the_repository->objects->sources; source; source = source->next) {
-		struct multi_pack_index *m = get_multi_pack_index(source);
+		struct odb_source_files *files = odb_source_files_downcast(source);
+		struct multi_pack_index *m = get_multi_pack_index(files->packed);
 		struct pack_entry e;
 
 		if (m && fill_midx_entry(m, oid, &e)) {
@@ -4506,8 +4507,8 @@ static void add_objects_in_unpacked_packs(void)
 		if (!source->local)
 			continue;
 
-		if (packfile_store_for_each_object(files->packed, &oi,
-						   add_object_in_unpacked_pack, NULL, &opts))
+		if (odb_source_for_each_object(&files->packed->base, &oi,
+					       add_object_in_unpacked_pack, NULL, &opts))
 			die(_("cannot open pack index"));
 	}
 }

builtin/repack.c

@@ -458,6 +458,8 @@ int cmd_repack(int argc,
 	}
 
 	if (!names.nr) {
+		struct odb_source_files *files = odb_source_files_downcast(existing.source);
+
 		if (!po_args.quiet)
 			printf_ln(_("Nothing new to pack."));
 		/*
@@ -473,7 +475,7 @@ int cmd_repack(int argc,
 		 * midx_has_unknown_packs() will make the decision for
 		 * us.
 		 */
-		if (!get_multi_pack_index(existing.source))
+		if (!get_multi_pack_index(files->packed))
 			midx_must_contain_cruft = 1;
 	}
 
@@ -626,10 +628,12 @@ int cmd_repack(int argc,
 		update_server_info(repo, 0);
 
 	if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) {
+		struct odb_source_files *files = odb_source_files_downcast(existing.source);
 		unsigned flags = 0;
+
 		if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0))
 			flags |= MIDX_WRITE_INCREMENTAL;
-		write_midx_file(existing.source, NULL, NULL, flags);
+		write_midx_file(files->packed, NULL, NULL, flags);
 	}
 
 cleanup:

commit-graph.c

@@ -2016,8 +2016,8 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx)
 	odb_prepare_alternates(ctx->r->objects);
 	for (source = ctx->r->objects->sources; source; source = source->next) {
 		struct odb_source_files *files = odb_source_files_downcast(source);
-		packfile_store_for_each_object(files->packed, &oi, add_packed_commits_oi,
-					       ctx, &opts);
+		odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi,
+					   ctx, &opts);
 	}
 
 	if (ctx->progress_done < ctx->approx_nr_objects)

meson.build

@@ -406,6 +406,7 @@ libgit_sources = [
   'odb/source-files.c',
   'odb/source-inmemory.c',
   'odb/source-loose.c',
+  'odb/source-packed.c',
   'odb/streaming.c',
   'odb/transaction.c',
   'oid-array.c',
@@ -421,6 +422,7 @@ libgit_sources = [
   'pack-revindex.c',
   'pack-write.c',
   'packfile.c',
+  'packfile-list.c',
   'pager.c',
   'parallel-checkout.c',
   'parse.c',

midx-write.c

@@ -25,9 +25,9 @@
 #define NO_PREFERRED_PACK (~((uint32_t)0))
 
 extern int midx_checksum_valid(struct multi_pack_index *m);
-extern void clear_midx_files_ext(struct odb_source *source, const char *ext,
+extern void clear_midx_files_ext(struct odb_source_packed *source, const char *ext,
 				 const char *keep_hash);
-extern void clear_incremental_midx_files_ext(struct odb_source *source,
+extern void clear_incremental_midx_files_ext(struct odb_source_packed *source,
 					     const char *ext,
 					     const struct strvec *keep_hashes);
 extern int cmp_idx_or_pack_name(const char *idx_or_pack_name,
@@ -119,7 +119,7 @@ struct write_midx_context {
 	struct string_list *to_include;
 
 	struct repository *repo;
-	struct odb_source *source;
+	struct odb_source_packed *source;
 };
 
 static uint32_t midx_pack_perm(struct write_midx_context *ctx,
@@ -1107,7 +1107,7 @@ static int link_midx_to_chain(struct multi_pack_index *m)
 	return ret;
 }
 
-static void clear_midx_files(struct odb_source *source,
+static void clear_midx_files(struct odb_source_packed *source,
 			     const struct strvec *hashes, unsigned incremental)
 {
 	/*
@@ -1237,7 +1237,7 @@ static int midx_hashcmp(const struct multi_pack_index *a,
 }
 
 struct write_midx_opts {
-	struct odb_source *source; /* non-optional */
+	struct odb_source_packed *source; /* non-optional */
 
 	struct string_list *packs_to_include;
 	struct string_list *packs_to_drop;
@@ -1253,7 +1253,7 @@ struct write_midx_opts {
 
 static int write_midx_internal(struct write_midx_opts *opts)
 {
-	struct repository *r = opts->source->odb->repo;
+	struct repository *r = opts->source->base.odb->repo;
 	struct strbuf midx_name = STRBUF_INIT;
 	unsigned char midx_hash[GIT_MAX_RAWSZ];
 	uint32_t start_pack;
@@ -1301,7 +1301,7 @@ static int write_midx_internal(struct write_midx_opts *opts)
 	if (ctx.incremental)
 		strbuf_addf(&midx_name,
 			    "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX",
-			    opts->source->path);
+			    opts->source->base.path);
 	else
 		get_midx_filename(opts->source, &midx_name);
 	if (safe_create_leading_directories(r, midx_name.buf))
@@ -1396,7 +1396,7 @@ static int write_midx_internal(struct write_midx_opts *opts)
 		fill_packs_from_midx_range(&ctx, bitmap_order);
 	} else {
 		ctx.to_include = opts->packs_to_include;
-		for_each_file_in_pack_dir(opts->source->path, add_pack_to_midx, &ctx);
+		for_each_file_in_pack_dir(opts->source->base.path, add_pack_to_midx, &ctx);
 	}
 	stop_progress(&ctx.progress);
 
@@ -1847,7 +1847,7 @@ static int write_midx_internal(struct write_midx_opts *opts)
 	return result;
 }
 
-int write_midx_file(struct odb_source *source,
+int write_midx_file(struct odb_source_packed *source,
 		    const char *preferred_pack_name,
 		    const char *refs_snapshot,
 		    unsigned flags)
@@ -1862,7 +1862,7 @@ int write_midx_file(struct odb_source *source,
 	return write_midx_internal(&opts);
 }
 
-int write_midx_file_only(struct odb_source *source,
+int write_midx_file_only(struct odb_source_packed *source,
 			 struct string_list *packs_to_include,
 			 const char *preferred_pack_name,
 			 const char *refs_snapshot,
@@ -1881,7 +1881,7 @@ int write_midx_file_only(struct odb_source *source,
 	return write_midx_internal(&opts);
 }
 
-int write_midx_file_compact(struct odb_source *source,
+int write_midx_file_compact(struct odb_source_packed *source,
 			    struct multi_pack_index *from,
 			    struct multi_pack_index *to,
 			    const char *incremental_base,
@@ -1898,7 +1898,7 @@ int write_midx_file_compact(struct odb_source *source,
 	return write_midx_internal(&opts);
 }
 
-int expire_midx_packs(struct odb_source *source, unsigned flags)
+int expire_midx_packs(struct odb_source_packed *source, unsigned flags)
 {
 	uint32_t i, *count, result = 0;
 	struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
@@ -1915,7 +1915,7 @@ int expire_midx_packs(struct odb_source *source, unsigned flags)
 
 	if (flags & MIDX_PROGRESS)
 		progress = start_delayed_progress(
-					  source->odb->repo,
+					  source->base.odb->repo,
 					  _("Counting referenced objects"),
 					  m->num_objects);
 	for (i = 0; i < m->num_objects; i++) {
@@ -1927,7 +1927,7 @@ int expire_midx_packs(struct odb_source *source, unsigned flags)
 
 	if (flags & MIDX_PROGRESS)
 		progress = start_delayed_progress(
-					  source->odb->repo,
+					  source->base.odb->repo,
 					  _("Finding and deleting unreferenced packfiles"),
 					  m->num_packs);
 	for (i = 0; i < m->num_packs; i++) {
@@ -2085,9 +2085,9 @@ static void fill_included_packs_batch(struct repository *r,
 	free(pack_info);
 }
 
-int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags)
+int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned flags)
 {
-	struct repository *r = source->odb->repo;
+	struct repository *r = source->base.odb->repo;
 	int result = 0;
 	uint32_t i, packs_to_repack = 0;
 	unsigned char *include_pack;
@@ -2131,7 +2131,7 @@ int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags)
 
 	strvec_push(&cmd.args, "pack-objects");
 
-	strvec_pushf(&cmd.args, "%s/pack/pack", source->path);
+	strvec_pushf(&cmd.args, "%s/pack/pack", source->base.path);
 
 	if (delta_base_offset)
 		strvec_push(&cmd.args, "--delta-base-offset");