fetch-pack: aggregate progress for parallel URI packs

Documentation/config/fetch.adoc

@@ -94,6 +94,16 @@ A value of 0 will give some reasonable default. If unset, it defaults to 1.
 For submodules, this setting can be overridden using the `submodule.fetchJobs`
 config setting.
 
+`fetch.packfileUriJobs`::
+	Specifies the maximum number of packfile URI downloads and indexers
+	to run at once. The default is 1, which preserves advertised URI
+	order.
++
+Values greater than 1 are used only when the server advertises and the
+client requests the `no-ref-delta` promise. Each URI pack is then checked
+with `index-pack --no-ref-delta` before it is accepted.
+Responses with one URI retain the serial path.
+
 `fetch.writeCommitGraph`::
 	Set to true to write a commit-graph after every `git fetch` command
 	that downloads a pack-file from a remote. Using the `--split` option,

Documentation/config/uploadpack.adoc

@@ -86,3 +86,14 @@ uploadpack.allowRefInWant::
 	is intended for the benefit of load-balanced servers which may
 	not have the same view of what OIDs their refs point to due to
 	replication delay.
+
+uploadpack.allowNoRefDelta::
+	If this option is set, `upload-pack` may advertise the
+	`no-ref-delta` feature of the protocol version 2 `fetch`
+	command. When a client requests the feature, `upload-pack` passes
+	`--no-ref-delta` to `pack-objects` for the inline pack and promises
+	that every pack named in a `packfile-uris` response section also
+	contains no `REF_DELTA` entries. `upload-pack` does not inspect
+	configured URI packs, so the server administrator must create each
+	pack with `pack-objects --no-ref-delta` or otherwise verify this
+	property. The default is `false`.

Documentation/git-http-fetch.adoc

@@ -48,13 +48,21 @@ commit-id::
 	line (which is not expected in
 	this case), 'git http-fetch' fetches the packfile directly at the given
 	URL and uses index-pack to generate corresponding .idx and .keep files.
-	The hash is used to determine the name of the temporary file and is
-	arbitrary. The output of index-pack is printed to stdout. Requires
-	--index-pack-args.
+	The hash is used to determine the name of the temporary file. It need
+	not be the pack hash, but it must uniquely identify the pack contents
+	for resumption. The output of index-pack is printed to stdout. Requires
+	one or more --index-pack-arg options.
 
---index-pack-args=<args>::
-	For internal use only. The command to run on the contents of the
-	downloaded pack. Arguments are URL-encoded separated by spaces.
+--index-pack-arg=<arg>::
+	For internal use only. The first instance specifies the command run on
+	the contents of the downloaded pack. Subsequent instances specify its
+	arguments.
+
+--report-progress::
+	For internal use only. With --packfile, report the cumulative number
+	of bytes received as `bytes <count>` lines on stdout, before the
+	index-pack output. Bytes already present in a resumed download are
+	not included.
 
 --recover::
 	Verify that everything reachable from target is fetched.  Used after

Documentation/git-index-pack.adoc

@@ -9,8 +9,10 @@ git-index-pack - Build pack index file for an existing packed archive
 SYNOPSIS
 --------
 [verse]
-'git index-pack' [-v] [-o <index-file>] [--[no-]rev-index] <pack-file>
-'git index-pack' --stdin [--fix-thin] [--keep] [-v] [-o <index-file>]
+'git index-pack' [-v] [-o <index-file>] [--no-ref-delta]
+		[--[no-]rev-index] <pack-file>
+'git index-pack' --stdin [--fix-thin] [--keep] [--no-ref-delta]
+		[-v] [-o <index-file>]
 		  [--[no-]rev-index] [<pack-file>]
 
 
@@ -60,6 +62,11 @@ OPTIONS
 	excluded objects the deltified objects are based on to the
 	pack. This option only makes sense in conjunction with --stdin.
 
+--no-ref-delta::
+	Reject a pack containing a `REF_DELTA` entry. `OFS_DELTA` entries
+	are accepted. This option can be used to verify a protocol promise
+	that a pack contains no `REF_DELTA` entries.
+
 --keep::
 	Before moving the index into its final destination
 	create an empty .keep file for the associated pack file.

Documentation/git-pack-objects.adoc

@@ -10,7 +10,8 @@ SYNOPSIS
 --------
 [verse]
 'git pack-objects' [-q | --progress | --all-progress] [--all-progress-implied]
-		   [--no-reuse-delta] [--delta-base-offset] [--non-empty]
+		   [--no-reuse-delta] [--delta-base-offset] [--no-ref-delta]
+		   [--non-empty]
 		   [--local] [--incremental] [--window=<n>] [--depth=<n>]
 		   [--revs [--unpacked | --all]] [--keep-pack=<pack-name>]
 		   [--cruft] [--cruft-expiration=<time>]
@@ -297,6 +298,11 @@ Note: Porcelain commands such as `git gc` (see linkgit:git-gc[1]),
 in modern Git when they put objects in your repository into pack files.
 So does `git bundle` (see linkgit:git-bundle[1]) when it creates a bundle.
 
+--no-ref-delta::
+	Do not emit deltas which represent their base by their literal
+	object ID. This is independent of `--delta-base-offset`;
+	without that option, no deltas are emitted.
+
 --threads=<n>::
 	Specifies the number of threads to spawn when searching for best
 	delta matches.  This requires that pack-objects be compiled with

Documentation/gitprotocol-capabilities.adoc

@@ -34,9 +34,9 @@ were sent.  Server MUST NOT ignore capabilities that client requested
 and server advertised.  As a consequence of these rules, server MUST
 NOT advertise capabilities it does not understand.
 
-The 'atomic', 'report-status', 'report-status-v2', 'delete-refs', 'quiet',
-and 'push-cert' capabilities are sent and recognized by the receive-pack
-(push to server) process.
+The 'atomic', 'report-status', 'report-status-v2', 'delete-refs',
+'no-ref-delta', 'quiet', and 'push-cert' capabilities are sent and
+recognized by the receive-pack (push to server) process.
 
 The 'ofs-delta' and 'side-band-64k' capabilities are sent and recognized
 by both upload-pack and receive-pack protocols.  The 'agent' and 'session-id'
@@ -174,6 +174,21 @@ The server can send, and the client can understand, PACKv2 with delta referring
 its base by position in pack rather than by an obj-id.  That is, they can
 send/read OBJ_OFS_DELTA (aka type 6) in a packfile.
 
+no-ref-delta
+------------
+
+The receive-pack server can request, and the client can send, PACKv2
+without deltas referring to their bases by an obj-id. That is, the
+client MUST NOT send OBJ_REF_DELTA (aka type 7) in a packfile when the
+server advertises this capability.
+
+This does not imply that the server understands OBJ_OFS_DELTA entries;
+that is negotiated separately with the 'ofs-delta' capability.
+
+Protocol v2 `fetch` uses the same name for the corresponding
+upload-pack request. There, the promise covers both the inline pack and
+any packs named by a `packfile-uris` section.
+
 agent
 -----
 

Documentation/gitprotocol-v2.adoc

@@ -376,6 +376,18 @@ to the server.
 	client should download from all given URIs. Currently, the
 	protocols supported are "http" and "https".
 
+If the 'no-ref-delta' feature is advertised, the following argument can
+be included in the client's request:
+
+    no-ref-delta
+	Indicates that the server MUST NOT include a `REF_DELTA` entry in
+	the inline `packfile` section or any pack named by a
+	`packfile-uris` section.
++
+`no-ref-delta` does not imply `ofs-delta`. The server may use `OFS_DELTA`
+only when the client also requests `ofs-delta`. By definition, an
+`OFS_DELTA` refers to an earlier entry in the same pack.
+
 If the 'wait-for-done' feature is advertised, the following argument
 can be included in the client's request.
 

Documentation/technical/packfile-uri.adoc

@@ -25,6 +25,13 @@ Clients should then download and index all the given URIs (in addition to
 downloading and indexing the packfile given in the `packfile` section of the
 response) before performing the connectivity check.
 
+URI packs are normally indexed in advertised order, since a later pack may
+contain a `REF_DELTA` whose base is installed by an earlier response pack.
+If the server advertises `no-ref-delta` and the client requests it, the
+promise covers the inline pack and every URI pack. A client may then index
+URI packs concurrently, while rejecting any pack that contains a
+`REF_DELTA`.
+
 Server design
 -------------
 

builtin/index-pack.c

@@ -33,7 +33,7 @@
 #include "strvec.h"
 
 static const char index_pack_usage[] =
-"git index-pack [-v] [-o <index-file>] [--keep | --keep=<msg>] [--[no-]rev-index] [--verify] [--strict[=<msg-id>=<severity>...]] [--fsck-objects[=<msg-id>=<severity>...]] (<pack-file> | --stdin [--fix-thin] [<pack-file>])";
+"git index-pack [-v] [-o <index-file>] [--keep | --keep=<msg>] [--[no-]rev-index] [--verify] [--strict[=<msg-id>=<severity>...]] [--fsck-objects[=<msg-id>=<severity>...]] [--no-ref-delta] (<pack-file> | --stdin [--fix-thin] [<pack-file>])";
 
 struct object_entry {
 	struct pack_idx_entry idx;
@@ -138,6 +138,7 @@ static int strict;
 static int do_fsck_object;
 static struct fsck_options fsck_options;
 static int verbose;
+static int no_ref_delta;
 static const char *progress_title;
 static int show_resolving_progress;
 static int show_stat;
@@ -550,6 +551,9 @@ static void *unpack_raw_entry(struct object_entry *obj,
 
 	switch (obj->type) {
 	case OBJ_REF_DELTA:
+		if (no_ref_delta)
+			bad_object(obj->idx.offset,
+				   _("REF_DELTA not allowed by --no-ref-delta"));
 		oidread(ref_oid, fill(the_hash_algo->rawsz),
 			the_repository->hash_algo);
 		use(the_hash_algo->rawsz);
@@ -1937,6 +1941,8 @@ int cmd_index_pack(int argc,
 				from_stdin = 1;
 			} else if (!strcmp(arg, "--fix-thin")) {
 				fix_thin_pack = 1;
+			} else if (!strcmp(arg, "--no-ref-delta")) {
+				no_ref_delta = 1;
 			} else if (skip_to_optional_arg(arg, "--strict", &arg)) {
 				strict = 1;
 				do_fsck_object = 1;

builtin/pack-objects.c

@@ -191,7 +191,8 @@ static inline void oe_set_delta_size(struct packing_data *pack,
 
 static const char *const pack_usage[] = {
 	N_("git pack-objects [-q | --progress | --all-progress] [--all-progress-implied]\n"
-	   "                 [--no-reuse-delta] [--delta-base-offset] [--non-empty]\n"
+	   "                 [--no-reuse-delta] [--delta-base-offset] [--no-ref-delta]\n"
+	   "                 [--non-empty]\n"
 	   "                 [--local] [--incremental] [--window=<n>] [--depth=<n>]\n"
 	   "                 [--revs [--unpacked | --all]] [--keep-pack=<pack-name>]\n"
 	   "                 [--cruft] [--cruft-expiration=<time>]\n"
@@ -222,6 +223,7 @@ static int ignore_packed_keep_in_core;
 static int ignore_packed_keep_in_core_open;
 static int ignore_packed_keep_in_core_has_cruft;
 static int allow_ofs_delta;
+static int allow_ref_delta = 1;
 static struct pack_idx_option pack_idx_opts;
 static const char *base_name;
 static int progress = 1;
@@ -2205,6 +2207,13 @@ static int can_reuse_delta(const struct object_id *base_oid,
 	 */
 	base = packlist_find(&to_pack, base_oid);
 	if (base) {
+		/*
+		 * A preferred base is omitted from the resulting pack, so it
+		 * can only be referenced by object ID.
+		 */
+		if (base->preferred_base && !allow_ref_delta)
+			return 0;
+
 		if (!in_same_island(&delta->idx.oid, &base->idx.oid))
 			return 0;
 		*base_out = base;
@@ -2216,7 +2225,8 @@ static int can_reuse_delta(const struct object_id *base_oid,
 	 * even if it was buried too deep in history to make it into the
 	 * packing list.
 	 */
-	if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
+	if (allow_ref_delta && thin &&
+	    bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) {
 		if (use_delta_islands) {
 			if (!in_same_island(&delta->idx.oid, base_oid))
 				return 0;
@@ -3405,6 +3415,9 @@ static int should_attempt_deltas(struct object_entry *entry)
 	if (entry->no_try_delta)
 		return 0;
 
+	if (entry->preferred_base && !allow_ref_delta)
+		return 0;
+
 	if (!entry->preferred_base) {
 		if (oe_type(entry) < 0)
 			die(_("unable to get type of object %s"),
@@ -3647,7 +3660,8 @@ static void prepare_pack(int window, int depth)
 	if (!pack_to_stdout)
 		do_check_packed_object_crc = 1;
 
-	if (!to_pack.nr_objects || !window || !depth)
+	if (!to_pack.nr_objects || !window || !depth ||
+	    (!allow_ref_delta && !allow_ofs_delta))
 		return;
 
 	if (path_walk)
@@ -4710,7 +4724,7 @@ static int pack_options_allow_reuse(void)
 	       !ignore_packed_keep_on_disk &&
 	       !ignore_packed_keep_in_core &&
 	       (!local || !have_non_local_packs) &&
-	       !incremental;
+	       !incremental && (allow_ref_delta || allow_ofs_delta);
 }
 
 static int get_object_list_from_bitmap(struct rev_info *revs)
@@ -4732,7 +4746,8 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
 						   &reuse_packfiles,
 						   &reuse_packfiles_nr,
 						   &reuse_packfile_bitmap,
-						   allow_pack_reuse == MULTI_PACK_REUSE);
+						   allow_pack_reuse == MULTI_PACK_REUSE,
+						   allow_ref_delta);
 
 	if (reuse_packfiles) {
 		reuse_packfile_objects = bitmap_popcount(reuse_packfile_bitmap);
@@ -5163,6 +5178,8 @@ int cmd_pack_objects(int argc,
 			 N_("reuse existing objects")),
 		OPT_BOOL(0, "delta-base-offset", &allow_ofs_delta,
 			 N_("use OFS_DELTA objects")),
+		OPT_BOOL(0, "ref-delta", &allow_ref_delta,
+			 N_("use REF_DELTA objects")),
 		OPT_INTEGER(0, "threads", &delta_search_threads,
 			    N_("use threads when searching for best delta matches")),
 		OPT_BOOL(0, "non-empty", &non_empty,
@@ -5361,7 +5378,7 @@ int cmd_pack_objects(int argc,
 	if (unpack_unreachable || keep_unreachable || pack_loose_unreachable)
 		use_internal_rev_list = 1;
 
-	if (!reuse_object)
+	if (!reuse_object || (!allow_ref_delta && !allow_ofs_delta))
 		reuse_delta = 0;
 	if (cfg->pack_compression_level == -1)
 		cfg->pack_compression_level = Z_DEFAULT_COMPRESSION;

builtin/receive-pack.c

@@ -65,6 +65,7 @@ static struct strbuf fsck_msg_types = STRBUF_INIT;
 static int receive_unpack_limit = -1;
 static int transfer_unpack_limit = -1;
 static int advertise_atomic_push = 1;
+static int advertise_no_ref_delta;
 static int advertise_push_options;
 static int advertise_sid;
 static int unpack_limit = 100;
@@ -287,6 +288,8 @@ static void show_ref(const char *path, const struct object_id *oid)
 			strbuf_addstr(&cap, " atomic");
 		if (prefer_ofs_delta)
 			strbuf_addstr(&cap, " ofs-delta");
+		if (advertise_no_ref_delta)
+			strbuf_addstr(&cap, " no-ref-delta");
 		if (push_cert_nonce)
 			strbuf_addf(&cap, " push-cert=%s", push_cert_nonce);
 		if (advertise_push_options)
@@ -2629,6 +2632,8 @@ int cmd_receive_pack(int argc,
 		OPT_HIDDEN_BOOL(0, "http-backend-info-refs", &advertise_refs, NULL),
 		OPT_ALIAS(0, "advertise-refs", "http-backend-info-refs"),
 		OPT_HIDDEN_BOOL(0, "reject-thin-pack-for-testing", &reject_thin, NULL),
+		OPT_HIDDEN_BOOL(0, "advertise-no-ref-delta-for-testing",
+				&advertise_no_ref_delta, NULL),
 		OPT_END()
 	};