diff --git a/.gitignore b/.gitignore index 4da58c6754899e..3db5d3635125b1 100644 --- a/.gitignore +++ b/.gitignore @@ -119,6 +119,7 @@ /git-name-rev /git-notes /git-p4 +/git-pack-aggregate /git-pack-redundant /git-pack-objects /git-pack-refs diff --git a/Documentation/config/pack.adoc b/Documentation/config/pack.adoc index 22384c2d2f0773..6c1e81112462a0 100644 --- a/Documentation/config/pack.adoc +++ b/Documentation/config/pack.adoc @@ -39,11 +39,26 @@ is set to "multi", reuse parts of just the bitmapped packfile. This can reduce memory and CPU usage to serve fetches, but might result in sending a slightly larger pack. Defaults to true. +pack.aggregateMaxObjects:: + Default for linkgit:git-pack-aggregate[1]'s `--max-objects` + option. Defaults to `100000`; see that option for details. + +pack.aggregateMaxInputPackSize:: + Default for linkgit:git-pack-aggregate[1]'s `--max-input-pack-size` + option. Defaults to `0` (automatic); see that option for details. + +pack.aggregateMaxLooseObjects:: + Default for linkgit:git-pack-aggregate[1]'s `--max-loose-objects` + option. Defaults to `100000`; see that option for details. + +pack.aggregateMaxPacks:: + Default for linkgit:git-pack-aggregate[1]'s `--max-packs` + option. Defaults to `10000`; see that option for details. + pack.island:: An extended regular expression configuring a set of delta islands. See "DELTA ISLANDS" in linkgit:git-pack-objects[1] for details. - pack.islandCore:: Specify an island name which gets to have its objects be packed first. This creates a kind of pseudo-pack at the front diff --git a/Documentation/config/repack.adoc b/Documentation/config/repack.adoc index 4c22a499f6216c..d2ecd6c00c6449 100644 --- a/Documentation/config/repack.adoc +++ b/Documentation/config/repack.adoc @@ -64,3 +64,21 @@ repack.midxNewLayerThreshold:: When the tip layer has fewer packs than this threshold, those packs are excluded from the geometric repack entirely, and are thus left unmodified. Must be at least 1. Defaults to 8. + +repack.aggregateOnce:: + If set to true, linkgit:git-repack[1] will run + linkgit:git-pack-aggregate[1] once before inspecting the packs + and loose objects to repack. This can quickly reduce a large + number of files before the more thorough repack begins. + Defaults to false. Can be overridden on the command line with + `--aggregate-once` or `--no-aggregate-once`. + +repack.aggregateLoop:: + If set to true, linkgit:git-repack[1] will spawn a background + linkgit:git-pack-aggregate[1] for the duration of its main + pack-objects run. The aggregator rolls up small packs and + loose objects that arrive after pack-objects has enumerated + its inputs, preventing them from accumulating in + `objects/pack/` and slowing other Git operations on busy + servers. Defaults to false. Can be overridden on the + command line with `--aggregate-loop` or `--no-aggregate-loop`. diff --git a/Documentation/git-pack-aggregate.adoc b/Documentation/git-pack-aggregate.adoc new file mode 100644 index 00000000000000..3762c88e2bd816 --- /dev/null +++ b/Documentation/git-pack-aggregate.adoc @@ -0,0 +1,202 @@ +git-pack-aggregate(1) +===================== + +NAME +---- +git-pack-aggregate - Quickly roll up loose objects and small packs +without doing any delta compression or search + +SYNOPSIS +-------- +[verse] +'git pack-aggregate' (--once | --loop) [--interval=] + [--min-loose=] [--min-packs=] + [--max-loose-objects=] + [--max-objects=] [--max-packs=] + [--max-input-pack-size=] + [--keep-pack=] + [--exclude-pack-file=] + [--exclude-loose-file=] + [--parent-pipe-fd=] + +DESCRIPTION +----------- + +`git pack-aggregate` rolls accumulated loose objects and small +packfiles into new packs. It can run one cycle as a quick recovery +step, or run cycles periodically to keep new material under control +while longer-running maintenance proceeds. Each cycle has two steps: + +1. Bundle local loose objects (minus any listed in + `--exclude-loose-file`) into new packs and unlink the loose copies. +2. Aggregate small local packs (minus any excluded by `--keep-pack` or + `--exclude-pack-file`, or those referenced by the multi-pack-index, + or those carrying a `.keep`, `.promisor`, `.mtimes`, or `.bitmap` + sidecar) into new packs and unlink the source packs. If step 1 + writes a single pack, it is naturally a candidate here and will + normally be folded into the step-2 output. Multiple loose-rollup + packs are left for a later cycle. + +No delta search is performed and no bitmaps or commit-graphs are updated. +Existing packed representations are reused where possible. When an +object has both delta and full-object representations in the input +packs, aggregation prefers the existing delta. All output packs are +written with `pack-objects --window=0 --mark-bad-deltas`, so each one +ships with a `.baddeltas` sidecar (see linkgit:gitformat-pack[5]). A +subsequent thorough repack (linkgit:git-repack[1]) honors that marker +and reconsiders intra-pack deltas at that time. + +The `pack.packSizeLimit` setting can split either step's output into +multiple packs. All outputs from a batch are installed before its +input packs or loose objects are removed. Size-based splitting may +require expanding deltas whose bases are in another output pack. +By default, step 2 skips packs larger than half that limit to avoid +repeatedly copying nearly full packs without reducing the pack count. +See `--max-input-pack-size` for a smaller input-size threshold. + +This command serves two related purposes. With `--once`, it provides a +quick recovery step for a repository that has accumulated many loose +objects or small packs. With `--loop`, it keeps new material under +control while a long-running repack is in flight. Both reduce the +number of files that unrelated Git operations must scan, while leaving +a later thorough linkgit:git-repack[1] to optimize deltas and pack +layout. The `repack.aggregateOnce` and `repack.aggregateLoop` +configuration variables (see linkgit:git-repack[1]) let `git repack` +request either behavior independently. + +Like linkgit:git-repack[1], `git pack-aggregate` takes no locks of its +own. Callers that need serialization must arrange it themselves (for +example by running under `git gc`). +Callers can declare "do not touch these inputs" with `--keep-pack`, +`--exclude-pack-file`, and `--exclude-loose-file`. +When spawned by `git repack`, protection across the parent's MIDX update +is layered: each aggregation cycle skips packs already named by the +current MIDX and packs with protective sidecars; `--emit-input-packs` +protects every local pack visible when the parent's `pack-objects` +prepares its inputs; and temporary `.keep` files protect the parent's new +packs before their indexes become visible and remain until after its MIDX +write. The parent's explicit MIDX include list is drawn from those +protected existing and newly-written packs, so the aggregator cannot +retire a pack that the parent is about to reference. + +OPTIONS +------- + +--once:: + Run a single cycle and exit. Exactly one of `--once` or + `--loop` is required. + +--loop:: + Run cycles forever, sleeping `--interval` seconds between + cycles. Stops on `SIGTERM`, `SIGHUP`, or `SIGINT`. + +--interval=:: + Number of seconds to sleep between the end of one cycle and the + start of the next. Defaults to 60. Only meaningful with + `--loop`. + +--min-loose=:: + Skip aggregation of loose objects if fewer than `` remain + after applying `--exclude-loose-file`. Defaults to 5. + +--min-packs=:: + Skip aggregation of small packfiles if fewer than `` + aggregatable packs remain after applying all exclusions. Defaults + to 5. + +--keep-pack=:: + Exclude the given pack from aggregation. `` is the pack + file name without a leading directory (e.g. `pack-123.pack`). + This option can be repeated to keep multiple packs. + +--exclude-pack-file=:: + Read a list of pack basenames (one per line) from `` and + never touch any of those packs. Lines may name a basename with + or without a `.pack` or `.idx` suffix; the suffix is stripped. + Blank lines and lines beginning with `#` are ignored. When + `git pack-aggregate` is spawned by a long-running `git repack`, + this file is populated by that `pack-objects` itself via + `--emit-input-packs`, and may contain a conservative superset + of the packs it will actually consume (such as in `--geometric` + mode). + +--exclude-loose-file=:: + Read a list of loose object IDs (one per line) from `` + and never pack or unlink any of those loose objects. Blank + lines and lines beginning with `#` are ignored. When `git + pack-aggregate` is spawned by a long-running `git repack`, this + file is populated by that `pack-objects` itself via + `--emit-input-loose`. + +--parent-pipe-fd=:: + An inherited pipe file descriptor whose write end the parent + process holds open. When the parent exits the pipe is closed + and `git pack-aggregate` exits at the next cycle boundary, + without waiting out the remainder of `--interval`. This is an + internal plumbing option used by `git repack` to keep its + companion aggregator from outliving it. + +--max-loose-objects=:: + Process at most `` loose objects per tranche. All output packs + are installed and the loose copies are removed before the next + tranche begins, so repository performance can improve incrementally + during recovery from an extreme backlog. Loose objects created + after the cycle starts are left for the next cycle. + If multiple packs are produced, they are left for a later aggregation + cycle instead of being copied again immediately. + Defaults to the `pack.aggregateMaxLooseObjects` configuration value, + or 100000 if that is unset. A value of `0` disables the limit. + +--max-objects=:: + Skip any pack that contains more than `` objects, leaving it + untouched by step 2. The object count is estimated cheaply from + the size of the pack's `.idx` file rather than by opening it. + This keeps `git pack-aggregate` focused on rolling up the small + packs it is meant for, instead of repacking a large base pack + (which would amount to a near-full repack). Defaults to the + `pack.aggregateMaxObjects` configuration value, or 100000 if that + is unset. A value of `0` disables the limit. Note that packs + already referenced by the multi-pack-index are excluded + regardless of this setting, so on a normally-maintained + repository the large base pack is skipped anyway. + +--max-input-pack-size=:: + Skip input packs whose `.pack` file is larger than ``. + This byte-size gate applies in addition to `--max-objects`; + it does not limit loose-object rollup or individual output packs. + The suffixes `k`, `m`, and `g` are supported. ++ +Defaults to `pack.aggregateMaxInputPackSize`, or `0` (automatic). +Automatic selection uses half of `pack.packSizeLimit`, after applying +the same 1-MiB minimum for nonzero output limits as `pack-objects`. +An explicit positive input limit above that ceiling is an error; +a lower limit is allowed. If `pack.packSizeLimit` is unset or zero, +automatic selection imposes no byte-size gate, and any explicit +positive input limit is allowed. An explicit `0` overrides the +configuration and restores automatic selection. + +--max-packs=:: + Process at most `` input packs per batch. When more than + `` aggregatable packs are present, they are split into several + evenly-sized batches, each producing one or more output packs. + This bounds how many packs any single `pack-objects` run (and any + later reader) must keep open at once, which matters on repositories + with such an extreme number of packs that per-process limits on + memory mappings or open files would otherwise be exceeded. Note + that a pack in use costs roughly three such resources -- its + `.idx`, `.pack`, and `.rev` files -- so a batch of `` packs + holds on the order of `3 * ` at peak; leave headroom below the + relevant limit when choosing ``. Batching happens on whole-pack + boundaries, keeping each delta and its base in the same input batch. + Defaults to the `pack.aggregateMaxPacks` configuration value, or + `10000` if unset. A value of `0` disables the input-pack limit and + processes all input packs in one batch. + +SEE ALSO +-------- +linkgit:git-repack[1], linkgit:git-pack-objects[1], +linkgit:gitformat-pack[5] + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-pack-objects.adoc b/Documentation/git-pack-objects.adoc index 65cd00c152f495..febd5c5e192151 100644 --- a/Documentation/git-pack-objects.adoc +++ b/Documentation/git-pack-objects.adoc @@ -143,6 +143,14 @@ options which imply `--revs`. have an mtime older than ``. If unspecified (and given `--cruft`), then no objects are eliminated. +--mark-bad-deltas:: + Write a `.baddeltas` marker file alongside each output pack. The + marker signals that objects within the pack have not been fully + delta-searched against other objects within the same pack and + that future repacking should consider them. Any deltas that do + exist within this pack can still be reused, however. + Incompatible with `--stdout`. + --window=:: --depth=:: These two options affect how the objects contained in @@ -246,6 +254,13 @@ depth is 4095. wholesale enforcement of a different compression level on the packed data is desired. +--prefer-reused-deltas:: + Only meaningful with `--stdin-packs`. When the same object is + present as a base in one included pack and a delta in another, + record the delta copy rather than the base. This costs a + cheap object-header read per duplicate, and has no effect + under `--no-reuse-delta`. + --compression=:: Specifies compression level for newly-compressed data in the generated pack. If not specified, pack compression level is diff --git a/Documentation/git-repack.adoc b/Documentation/git-repack.adoc index a1f9e64f668750..8236bd8c06651f 100644 --- a/Documentation/git-repack.adoc +++ b/Documentation/git-repack.adoc @@ -13,6 +13,7 @@ SYNOPSIS [--window=] [--depth=] [--threads=] [--keep-pack=] [--write-midx[=]] [--name-hash-version=] [--path-walk] [--filter=] [--drop-filtered [--dry-run]] + [--[no-]aggregate-once] [--[no-]aggregate-loop] DESCRIPTION ----------- @@ -274,6 +275,9 @@ picks the smallest set of packfiles such that as many of the larger packfiles (by count of objects contained in that pack) may be left intact. + +Selection also accounts for packs forced into the roll-up by `.baddeltas` +markers, including another pack if needed to preserve the progression. ++ Unlike other repack modes, the set of objects to pack is determined uniquely by the set of packs being "rolled-up"; in other words, the packs determined to need to be combined in order to restore a geometric @@ -337,6 +341,26 @@ created for any new pack(s) without disturbing the existing chain. Pass the `--path-walk` option to the underlying `git pack-objects` process. See linkgit:git-pack-objects[1] for full details. +--aggregate-once:: +--no-aggregate-once:: + Run linkgit:git-pack-aggregate[1] once before inspecting the + packs and loose objects to repack. This can quickly reduce a + large number of files before the more thorough repack begins. + Packs named by `--keep-pack` are excluded from this preliminary pass. + Overrides the `repack.aggregateOnce` configuration variable. + Off by default. + +--aggregate-loop:: +--no-aggregate-loop:: + Spawn a background linkgit:git-pack-aggregate[1] for the + duration of the main pack-objects run. The aggregator rolls + up small packs and loose objects that arrive after + pack-objects has enumerated its inputs, preventing them from + accumulating in `objects/pack/` and slowing other Git + operations on busy servers. Overrides the + `repack.aggregateLoop` configuration variable. Off by + default. + CONFIGURATION ------------- @@ -361,6 +385,7 @@ SEE ALSO -------- linkgit:git-pack-objects[1] linkgit:git-prune-packed[1] +linkgit:git-pack-aggregate[1] GIT --- diff --git a/Documentation/gitformat-pack.adoc b/Documentation/gitformat-pack.adoc index 3416edceab82e9..6ef84c9cefc8dc 100644 --- a/Documentation/gitformat-pack.adoc +++ b/Documentation/gitformat-pack.adoc @@ -12,6 +12,7 @@ SYNOPSIS $GIT_DIR/objects/pack/pack-*.{pack,idx} $GIT_DIR/objects/pack/pack-*.rev $GIT_DIR/objects/pack/pack-*.mtimes +$GIT_DIR/objects/pack/pack-*.baddeltas $GIT_DIR/objects/pack/multi-pack-index DESCRIPTION @@ -357,6 +358,35 @@ All 4-byte numbers are in network byte order. and a checksum of all of the above (each having length according to the specified hash function). +== pack-*.baddeltas files + +The optional `.baddeltas` file is an empty marker sitting alongside a +`pack-*.pack` (and its `.idx`). It signals to `git pack-objects` that +the delta layout of the pack should not be trusted: even when two +objects appear together in the same pack and neither is stored as a +delta, the next packing run should still call out to its delta search +routine for the pair instead of assuming a prior pack-objects already +considered (and rejected) the pair. + +This is intended for producers that intentionally skip delta search +when writing a pack (for example, processes that bulk-import objects +or aggregate multiple existing packs without recomputing deltas). +Without this marker, the same-pack delta skip in `git pack-objects` +would silently inherit those producers' lack of delta search into +future repacks. + +The contents of the file are currently ignored. Producers should +write an empty file; consumers must tolerate (and ignore) any +content. + +When `git repack` replaces a marked pack with an unmarked one, it +removes the old marker even if the pack's name is unchanged. + +The marker only affects whether `git pack-objects` will attempt to +compute new deltas for object pairs that share the marked pack. It +does not disable reuse of existing on-disk deltas, nor does it affect +multi-pack-index, bitmap, or pack reuse for transfer. + == multi-pack-index (MIDX) files have the following format: The multi-pack-index files refer to multiple pack-files and loose objects. diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..c1c7d2bf1ea68b 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -99,6 +99,7 @@ manpages = { 'git-name-rev.adoc' : 1, 'git-notes.adoc' : 1, 'git-p4.adoc' : 1, + 'git-pack-aggregate.adoc' : 1, 'git-pack-objects.adoc' : 1, 'git-pack-refs.adoc' : 1, 'git-patch-id.adoc' : 1, diff --git a/Makefile b/Makefile index d4b775953d3842..179ce2b33b2161 100644 --- a/Makefile +++ b/Makefile @@ -1465,6 +1465,7 @@ BUILTIN_OBJS += builtin/multi-pack-index.o BUILTIN_OBJS += builtin/mv.o BUILTIN_OBJS += builtin/name-rev.o BUILTIN_OBJS += builtin/notes.o +BUILTIN_OBJS += builtin/pack-aggregate.o BUILTIN_OBJS += builtin/pack-objects.o ifndef WITH_BREAKING_CHANGES BUILTIN_OBJS += builtin/pack-redundant.o diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..8fdd14b49360cf 100644 --- a/builtin.h +++ b/builtin.h @@ -224,6 +224,7 @@ int cmd_multi_pack_index(int argc, const char **argv, const char *prefix, struct int cmd_mv(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_name_rev(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_notes(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_pack_aggregate(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_pack_objects(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_pack_redundant(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_patch_id(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/pack-aggregate.c b/builtin/pack-aggregate.c new file mode 100644 index 00000000000000..90c8c56cad2f3f --- /dev/null +++ b/builtin/pack-aggregate.c @@ -0,0 +1,871 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "builtin.h" +#include "config.h" +#include "dir.h" +#include "gettext.h" +#include "hash.h" +#include "hex.h" +#include "midx.h" +#include "object-file.h" +#include "odb.h" +#include "oid-array.h" +#include "packfile.h" +#include "parse-options.h" +#include "path.h" +#include "repository.h" +#include "run-command.h" +#include "sigchain.h" +#include "strbuf.h" +#include "string-list.h" +#include "strmap.h" +#include "strvec.h" +#include "tempfile.h" +#include "trace2.h" +#include "wrapper.h" + +static const char *const pack_aggregate_usage[] = { + N_("git pack-aggregate (--once | --loop) [--interval=]\n" + " [--min-loose=] [--min-packs=]\n" + " [--max-loose-objects=]\n" + " [--max-objects=] [--max-packs=]\n" + " [--max-input-pack-size=]\n" + " [--keep-pack=]\n" + " [--exclude-pack-file=]\n" + " [--exclude-loose-file=]\n" + " [--parent-pipe-fd=]"), + NULL +}; + +#define DEFAULT_MAX_LOOSE_OBJECTS 100000 + +#define DEFAULT_MAX_OBJECTS 100000 + +#define DEFAULT_MAX_PACKS 10000 + +static volatile sig_atomic_t stop_signaled; +static int parent_pipe_fd = -1; + +static void term_handler(int sig UNUSED) +{ + stop_signaled = 1; +} + +static int has_sidecar(const char *packdir, const char *basename, + const char *ext) +{ + struct strbuf buf = STRBUF_INIT; + struct stat st; + int ret; + + strbuf_addf(&buf, "%s/%s.%s", packdir, basename, ext); + ret = !lstat(buf.buf, &st); + strbuf_release(&buf); + return ret; +} + +static int has_protective_sidecar(const char *packdir, const char *basename) +{ + /* + * Ignore packs with sidecars that mean "don't touch me". .baddeltas + * is intentionally absent: rolling those up is the point. + */ + static const char *exts[] = { + "keep", "promisor", "mtimes", "bitmap", NULL + }; + int i; + + for (i = 0; exts[i]; i++) + if (has_sidecar(packdir, basename, exts[i])) + return 1; + return 0; +} + +static int idx_file_size(const char *packdir, const char *basename, + off_t *size) +{ + struct strbuf buf = STRBUF_INIT; + struct stat st; + int ret; + + strbuf_addf(&buf, "%s/%s.idx", packdir, basename); + ret = !lstat(buf.buf, &st); + if (ret) + *size = st.st_size; + strbuf_release(&buf); + return ret; +} + +/* + * Enumeration and index-building work scale with object count. Convert + * the cap to a maximum v2 index size to avoid opening or mapping candidate + * indexes. The v2 index is linear in the number of objects N: + * + * size = 8 (header) + 1024 (fanout) + N*(rawsz + 8) + 2*rawsz (trailer) + * + * Reusing the lstat() performed by idx_file_size() is much faster than + * opening, mapping, and reading each candidate index. In particular, + * p->num_objects is populated only by open_pack_index(), which mmaps the + * index; mapping every candidate merely to apply this gate can also exhaust + * vm.max_map_count on repositories with an enormous number of packs. + * + * The format also uses an 8-byte entry per large offset (objects at pack + * offset >= 2GiB), which we ignore here. Ignoring it makes our estimated N + * slightly high, i.e. we err on the side of skipping a borderline pack, + * which is the safe direction for this "don't touch large packs" gate. A + * cap of 0 disables the limit and is represented by a returned size of 0. + */ +static off_t max_objects_to_idx_size(const struct git_hash_algo *algo, + int max_objects) +{ + off_t rawsz = algo->rawsz; + + if (!max_objects) + return 0; + return 8 + 1024 + 2 * rawsz + (rawsz + 8) * (off_t)max_objects; +} + +static void load_exclusions_from_file(const char *path, struct strset *set) +{ + FILE *fp; + struct strbuf line = STRBUF_INIT; + + fp = fopen(path, "r"); + if (!fp) + die_errno(_("could not open exclude file '%s'"), path); + + while (strbuf_getline_lf(&line, fp) != EOF) { + strbuf_trim(&line); + if (!line.len || line.buf[0] == '#') + continue; + strbuf_strip_suffix(&line, ".pack"); + strbuf_strip_suffix(&line, ".idx"); + strset_add(set, line.buf); + } + strbuf_release(&line); + fclose(fp); +} + +/* + * Re-read the multi-pack-index (and any incremental layers) and + * populate `set` with the basenames of every referenced pack. The + * strset is cleared first so this is safe to call once per cycle. + */ +static int refresh_midx_exclusions(struct repository *repo, + struct strset *set) +{ + struct odb_source *source; + int had_midx = 0; + + strset_clear(set); + + odb_reprepare(repo->objects); + for (source = repo->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + struct multi_pack_index *m = get_multi_pack_index(files->packed); + for (; m; m = m->base_midx) { + uint32_t i; + had_midx = 1; + for (i = 0; i < m->num_packs; i++) { + struct strbuf base = STRBUF_INIT; + strbuf_addstr(&base, m->pack_names[i]); + strbuf_strip_suffix(&base, ".idx"); + strbuf_strip_suffix(&base, ".pack"); + strset_add(set, base.buf); + strbuf_release(&base); + } + } + } + return had_midx; +} + +/* ---------- loose-object pre-pass ---------- */ + +struct loose_scan { + struct strset *exclude; + struct oid_array *oids; + struct string_list *paths; + size_t limit; + size_t minimum; + size_t eligible; + time_t cutoff_sec; + unsigned int cutoff_nsec; +}; + +static void set_loose_scan_cutoff(struct repository *repo, + struct loose_scan *data) +{ + struct strbuf template = STRBUF_INIT; + struct tempfile *marker; + struct stat st; + + strbuf_addf(&template, "%s/.tmp-pack-aggregate-cutoff-XXXXXX", + repo_get_object_directory(repo)); + marker = xmks_tempfile(template.buf); + strbuf_release(&template); + + if (fstat(get_tempfile_fd(marker), &st)) + die_errno(_("could not stat loose-object cutoff marker")); + data->cutoff_sec = st.st_mtime; + data->cutoff_nsec = ST_MTIME_NSEC(st); + delete_tempfile(&marker); +} + +static int loose_scan_cb(const struct object_id *oid, const char *path, + void *cb_data) +{ + struct loose_scan *data = cb_data; + struct stat st; + + if (strset_contains(data->exclude, oid_to_hex(oid))) + return 0; + if (lstat(path, &st)) { + if (errno != ENOENT) + warning_errno(_("could not stat loose object '%s'"), + path); + return 0; + } + if (st.st_mtime > data->cutoff_sec || + (st.st_mtime == data->cutoff_sec && + ST_MTIME_NSEC(st) >= data->cutoff_nsec)) + return 0; + + data->eligible++; + if (!data->limit || data->oids->nr < data->limit) { + oid_array_append(data->oids, oid); + string_list_append(data->paths, path); + } + + if (data->limit && data->oids->nr >= data->limit && + data->eligible >= data->minimum) + return 1; + return 0; +} + +static int run_pack_objects(const char *packtmp, int stdin_packs, + const struct strbuf *input, + struct string_list *out_hashes) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf output = STRBUF_INIT; + int ret; + + strvec_push(&cmd.args, "pack-objects"); + if (stdin_packs) + strvec_pushl(&cmd.args, "--stdin-packs", + "--prefer-reused-deltas", NULL); + strvec_pushl(&cmd.args, + "--window=0", + "--mark-bad-deltas", + "--delta-base-offset", + "--no-write-bitmap-index", + "--quiet", + packtmp, + NULL); + cmd.git_cmd = 1; + cmd.clean_on_exit = 1; + + /* + * pipe_command() pumps stdin and stdout concurrently. Its + * clean-on-exit handling also terminates the child if we exit + * before it does. + */ + ret = pipe_command(&cmd, input->buf, input->len, &output, 0, NULL, 0); + + if (!ret && output.len) { + strbuf_strip_suffix(&output, "\n"); + string_list_split(out_hashes, output.buf, "\n", -1); + } + + strbuf_release(&output); + return ret; +} + +static int run_pack_objects_loose(const char *packtmp, struct oid_array *oids, + struct string_list *out_hashes) +{ + struct strbuf input = STRBUF_INIT; + size_t i; + int ret; + + for (i = 0; i < oids->nr; i++) + strbuf_addf(&input, "%s\n", oid_to_hex(&oids->oid[i])); + ret = run_pack_objects(packtmp, 0, &input, out_hashes); + strbuf_release(&input); + return ret; +} + +static int unlink_loose_paths(const struct string_list *paths) +{ + size_t i; + + for (i = 0; i < paths->nr; i++) + if (unlink_or_warn(paths->items[i].string)) + return -1; + return 0; +} + +/* ---------- pack aggregation ---------- */ + +/* + * True only for a durably installed "pack-" basename. This + * rejects an in-flight ".tmp--pack-" staging file written by + * a concurrent repack or pack-aggregate, which the object-store scan + * (matching any "*.idx") would otherwise surface as a candidate. + */ +static int is_canonical_pack_base(const char *base) +{ + const char *hex; + size_t i, hexsz = the_hash_algo->hexsz; + + if (!skip_prefix(base, "pack-", &hex)) + return 0; + for (i = 0; i < hexsz; i++) + if (!isxdigit(hex[i])) + return 0; + return hex[hexsz] == '\0'; +} + +static void collect_pack_candidates(struct repository *repo, + const char *packdir, + const struct string_list *keep_pack_list, + struct strset *file_exclude, + struct strset *cycle_exclude, + struct strset *midx_exclude, + struct string_list *candidates, + off_t max_idx_size, + unsigned long max_input_pack_size) +{ + struct packed_git *p; + struct strbuf base = STRBUF_INIT; + off_t idx_size = 0; + + repo_for_each_pack(repo, p) { + if (!p->pack_local) + continue; + if (max_input_pack_size && + (uintmax_t)p->pack_size > max_input_pack_size) + continue; + + strbuf_reset(&base); + strbuf_addstr(&base, pack_basename(p)); + if (string_list_has_string(keep_pack_list, base.buf)) + continue; + if (!strbuf_strip_suffix(&base, ".pack")) + continue; + + if (!is_canonical_pack_base(base.buf)) + continue; + + if (strset_contains(file_exclude, base.buf)) + continue; + if (strset_contains(cycle_exclude, base.buf)) + continue; + if (strset_contains(midx_exclude, base.buf)) + continue; + if (has_protective_sidecar(packdir, base.buf)) + continue; + if (!idx_file_size(packdir, base.buf, &idx_size)) + continue; + if (max_idx_size && idx_size > max_idx_size) + continue; + + string_list_append(candidates, base.buf); + } + + strbuf_release(&base); +} + +static int run_pack_objects_packs(const char *packtmp, + const struct string_list *bases, + size_t begin, size_t count, + struct string_list *out_hashes) +{ + struct strbuf input = STRBUF_INIT; + size_t i; + int ret; + + for (i = begin; i < begin + count; i++) + strbuf_addf(&input, "%s.pack\n", bases->items[i].string); + ret = run_pack_objects(packtmp, 1, &input, out_hashes); + strbuf_release(&input); + return ret; +} + +/* + * pack-objects writes its output as -.; rename it + * into place as /pack-.. .idx is renamed last so + * a concurrent reader scanning the pack directory never sees a .idx + * without its companion .pack. + */ +static int install_pack(struct repository *repo, const char *packtmp, + const char *packdir, const char *hash) +{ + static const char *exts[] = { + ".pack", ".rev", ".baddeltas", ".idx" + }; + struct tempfile *files[ARRAY_SIZE(exts)] = { 0 }; + size_t i; + + for (i = 0; i < ARRAY_SIZE(exts); i++) { + struct strbuf src = STRBUF_INIT; + struct stat st; + + strbuf_addf(&src, "%s-%s%s", packtmp, hash, exts[i]); + if (!stat(src.buf, &st)) { + files[i] = register_tempfile(src.buf); + if (adjust_shared_perm(repo, src.buf)) { + error_errno(_("unable to adjust permissions for '%s'"), + src.buf); + strbuf_release(&src); + goto cleanup; + } + } else if (errno != ENOENT) { + error_errno(_("could not stat '%s'"), src.buf); + strbuf_release(&src); + goto cleanup; + } + strbuf_release(&src); + } + + for (i = 0; i < ARRAY_SIZE(exts); i++) { + struct strbuf dst = STRBUF_INIT; + + if (!files[i]) + continue; + + strbuf_addf(&dst, "%s/pack-%s%s", packdir, hash, exts[i]); + if (rename_tempfile(&files[i], dst.buf)) { + error_errno(_("renaming pack to '%s' failed"), dst.buf); + strbuf_release(&dst); + goto cleanup; + } + strbuf_release(&dst); + } + return 0; + +cleanup: + for (i = 0; i < ARRAY_SIZE(files); i++) + if (files[i]) + delete_tempfile(&files[i]); + return -1; +} + +static int install_packs(struct repository *repo, const char *packtmp, + const char *packdir, + const struct string_list *hashes, + struct strset *output_bases) +{ + struct string_list_item *item; + struct strbuf base = STRBUF_INIT; + int ret = 0; + + for_each_string_list_item(item, hashes) { + struct object_id oid; + + if (strlen(item->string) != repo->hash_algo->hexsz || + get_oid_hex_algop(item->string, &oid, repo->hash_algo)) + return error(_("pack-objects returned an invalid pack hash")); + } + + for_each_string_list_item(item, hashes) { + if (install_pack(repo, packtmp, packdir, item->string)) { + ret = -1; + break; + } + strbuf_reset(&base); + strbuf_addf(&base, "pack-%s", item->string); + strset_add(output_bases, base.buf); + } + + strbuf_release(&base); + return ret; +} + +static void unlink_consumed_packs(const char *packdir, + const struct string_list *bases, + size_t begin, size_t count, + struct strset *keep_basenames) +{ + static const char *exts[] = { + "pack", "idx", "rev", "baddeltas", NULL + }; + size_t i; + + for (i = begin; i < begin + count; i++) { + const char *base = bases->items[i].string; + int j; + + /* + * Recheck protective sidecars: a .keep (or similar) may + * have appeared between scan and now, in which case the + * pack must stay. + */ + if (has_protective_sidecar(packdir, base)) + continue; + /* + * An output can be byte-identical to an input, so deleting + * that input would delete the output too. + */ + if (strset_contains(keep_basenames, base)) + continue; + + for (j = 0; exts[j]; j++) { + struct strbuf fname = STRBUF_INIT; + strbuf_addf(&fname, "%s/%s.%s", packdir, base, + exts[j]); + if (unlink(fname.buf) < 0 && errno != ENOENT) + warning_errno(_("could not unlink '%s'"), + fname.buf); + strbuf_release(&fname); + } + } +} + +static int do_one_cycle(struct repository *repo, const char *packdir, + const struct string_list *keep_pack_list, + struct strset *pack_exclude, + struct strset *loose_exclude, + struct strset *midx_exclude, + int min_loose, int min_packs, + int max_loose_objects, int max_objects, + int max_packs, unsigned long max_input_pack_size) +{ + struct oid_array loose_oids = OID_ARRAY_INIT; + struct string_list loose_paths = STRING_LIST_INIT_DUP; + struct loose_scan loose_data = { + .exclude = loose_exclude, + .oids = &loose_oids, + .paths = &loose_paths, + .limit = max_loose_objects, + .minimum = min_loose, + }; + struct strset loose_rollup_exclude = STRSET_INIT; + struct strset output_bases = STRSET_INIT; + struct string_list candidates = STRING_LIST_INIT_DUP; + struct string_list output_hashes = STRING_LIST_INIT_DUP; + char *packtmp_loose = NULL; + char *packtmp_packs = NULL; + int ret = 0; + + /* + * Step 1: pack loose objects in bounded tranches, installing every + * output before removing inputs. The cycle-start cutoff prevents + * new arrivals from prolonging this cycle. + */ + set_loose_scan_cutoff(repo, &loose_data); + for_each_loose_file_in_source(repo->objects->sources, + loose_scan_cb, NULL, NULL, &loose_data); + if (loose_data.eligible >= (size_t)min_loose && !stop_signaled) { + packtmp_loose = mkpathdup("%s/.tmp-%d-loose-pack", + packdir, (int)getpid()); + while (loose_oids.nr && !stop_signaled) { + string_list_clear(&output_hashes, 0); + if (run_pack_objects_loose(packtmp_loose, &loose_oids, + &output_hashes)) { + ret = error(_("pack-objects failed during " + "loose-object rollup")); + goto out; + } + if (!output_hashes.nr) + break; + + if (install_packs(repo, packtmp_loose, packdir, + &output_hashes, &loose_rollup_exclude) || + unlink_loose_paths(&loose_paths)) { + ret = -1; + goto out; + } + + oid_array_clear(&loose_oids); + string_list_clear(&loose_paths, 0); + if (stop_signaled) + break; + + /* Once rollup starts, also consume a final partial tranche. */ + loose_data.minimum = 1; + loose_data.eligible = 0; + for_each_loose_file_in_source(repo->objects->sources, + loose_scan_cb, NULL, NULL, + &loose_data); + } + } + + if (strset_get_size(&loose_rollup_exclude) == 1) + strset_clear(&loose_rollup_exclude); + + if (stop_signaled) + goto out; + + /* + * Step 2: aggregate small packs. Let a single loose-rollup output + * participate, but defer multiple outputs to avoid copying them twice. + * Refresh MIDX exclusions before collecting candidates. + */ + refresh_midx_exclusions(repo, midx_exclude); + collect_pack_candidates(repo, packdir, keep_pack_list, pack_exclude, + &loose_rollup_exclude, midx_exclude, &candidates, + max_objects_to_idx_size(repo->hash_algo, + max_objects), + max_input_pack_size); + + if ((int)candidates.nr < min_packs) + goto out; + + packtmp_packs = mkpathdup("%s/.tmp-%d-pack", + packdir, (int)getpid()); + + /* + * Batch whole packs so each input batch contains its delta bases. + * Aim for balanced sizes with ceil(total / ceil(total / max_packs)); + * max_packs == 0 means one batch containing everything. + */ + { + size_t total = candidates.nr; + size_t batch_size = total; + size_t start; + + if (max_packs > 0 && total > (size_t)max_packs) { + size_t num_batches = DIV_ROUND_UP(total, + (size_t)max_packs); + batch_size = DIV_ROUND_UP(total, num_batches); + } + + for (start = 0; start < total && !stop_signaled; + start += batch_size) { + size_t count = batch_size; + + if (start + count > total) + count = total - start; + + string_list_clear(&output_hashes, 0); + strset_clear(&output_bases); + if (run_pack_objects_packs(packtmp_packs, &candidates, + start, count, &output_hashes)) { + ret = error(_("pack-objects failed during " + "pack aggregation")); + goto out; + } + if (output_hashes.nr) { + if (install_packs(repo, packtmp_packs, packdir, + &output_hashes, &output_bases)) { + ret = -1; + goto out; + } + unlink_consumed_packs(packdir, &candidates, + start, count, + &output_bases); + } + } + } + +out: + free(packtmp_loose); + free(packtmp_packs); + string_list_clear(&output_hashes, 0); + strset_clear(&loose_rollup_exclude); + strset_clear(&output_bases); + string_list_clear(&candidates, 0); + oid_array_clear(&loose_oids); + string_list_clear(&loose_paths, 0); + return ret; +} + +static void interruptible_sleep(unsigned int seconds) +{ + struct pollfd pfd; + int timeout_ms; + + if (stop_signaled) + return; + + if (parent_pipe_fd < 0) { + unsigned int remaining = seconds; + while (remaining > 0 && !stop_signaled) + remaining = sleep(remaining); + return; + } + + /* + * Watch the parent pipe so we wake immediately if the process + * that spawned us exits. POLLHUP is reported in revents + * regardless of whether it appears in events, so we leave + * events==0; any of POLLHUP/POLLERR/POLLNVAL/POLLIN means the + * other end of the pipe is gone and we should stop. + */ + pfd.fd = parent_pipe_fd; + pfd.events = 0; + timeout_ms = (seconds > INT_MAX / 1000) ? INT_MAX + : (int)(seconds * 1000); + + while (!stop_signaled) { + int ret; + pfd.revents = 0; + ret = poll(&pfd, 1, timeout_ms); + if (ret < 0) { + if (errno == EINTR) + continue; + break; + } + if (ret == 0) + break; + if (pfd.revents & (POLLHUP | POLLERR | POLLNVAL | POLLIN)) { + stop_signaled = 1; + break; + } + } +} + +int cmd_pack_aggregate(int argc, const char **argv, + const char *prefix, struct repository *repo) +{ + const char *exclude_pack_file = NULL; + const char *exclude_loose_file = NULL; + int interval = 60; + int min_packs = 5; + int min_loose = 5; + int max_loose_objects = -1; + int max_objects = -1; + int max_packs = -1; + unsigned long max_input_pack_size = 0; + unsigned long pack_size_limit = 0; + struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; + int once = 0; + int loop = 0; + uintmax_t cycle_count = 0; + struct option options[] = { + OPT_BOOL(0, "once", &once, + N_("run a single cycle and exit")), + OPT_BOOL(0, "loop", &loop, + N_("loop forever, sleeping --interval seconds " + "between cycles")), + OPT_INTEGER(0, "interval", &interval, + N_("seconds to sleep between cycles " + "(default 60)")), + OPT_INTEGER(0, "min-loose", &min_loose, + N_("skip loose-object rollup if fewer " + "candidates (default 5)")), + OPT_INTEGER(0, "min-packs", &min_packs, + N_("skip pack aggregation if fewer " + "candidates (default 5)")), + OPT_INTEGER(0, "max-loose-objects", &max_loose_objects, + N_("pack at most this many loose objects per " + "tranche (0 for no limit)")), + OPT_INTEGER(0, "max-objects", &max_objects, + N_("skip packs with more than this many " + "objects (0 for no limit)")), + OPT_INTEGER(0, "max-packs", &max_packs, + N_("aggregate at most this many input packs per " + "batch (0 for no limit)")), + OPT_UNSIGNED(0, "max-input-pack-size", &max_input_pack_size, + N_("skip packs larger than this many bytes " + "(0 for automatic)")), + OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"), + N_("exclude the given pack from aggregation")), + OPT_STRING(0, "exclude-pack-file", &exclude_pack_file, + N_("file"), + N_("file listing pack basenames never to " + "touch")), + OPT_STRING(0, "exclude-loose-file", &exclude_loose_file, + N_("file"), + N_("file listing loose object OIDs never to " + "touch")), + OPT_INTEGER(0, "parent-pipe-fd", &parent_pipe_fd, + N_("inherited fd of a pipe whose write end " + "the parent holds; EOF triggers exit")), + OPT_END(), + }; + struct strset pack_exclude = STRSET_INIT; + struct strset loose_exclude = STRSET_INIT; + struct strset midx_exclude = STRSET_INIT; + char *packdir; + int ret = 0; + + if (repo) + repo_config_get_ulong(repo, "pack.aggregatemaxinputpacksize", + &max_input_pack_size); + + argc = parse_options(argc, argv, prefix, options, + pack_aggregate_usage, 0); + if (argc > 0) + usage_with_options(pack_aggregate_usage, options); + if (once == loop) + die(_("exactly one of --once or --loop is required")); + if (interval < 1) + die(_("--interval must be at least 1")); + if (min_loose < 1) + die(_("--min-loose must be at least 1")); + if (min_packs < 1) + die(_("--min-packs must be at least 1")); + + if (max_loose_objects < 0 && + repo_config_get_int(repo, "pack.aggregatemaxlooseobjects", + &max_loose_objects)) + max_loose_objects = DEFAULT_MAX_LOOSE_OBJECTS; + if (max_loose_objects < 0) + die(_("pack.aggregateMaxLooseObjects cannot be negative")); + + if (max_objects < 0 && + repo_config_get_int(repo, "pack.aggregatemaxobjects", &max_objects)) + max_objects = DEFAULT_MAX_OBJECTS; + if (max_objects < 0) + die(_("pack.aggregateMaxObjects cannot be negative")); + + if (max_packs < 0 && + repo_config_get_int(repo, "pack.aggregatemaxpacks", &max_packs)) + max_packs = DEFAULT_MAX_PACKS; + if (max_packs < 0) + die(_("pack.aggregateMaxPacks cannot be negative")); + + repo_config_get_ulong(repo, "pack.packsizelimit", &pack_size_limit); + /* Match pack-objects' minimum nonzero output size limit. */ + if (pack_size_limit && pack_size_limit < 1024 * 1024) + pack_size_limit = 1024 * 1024; + if (pack_size_limit) { + unsigned long ceiling = pack_size_limit / 2; + + if (max_input_pack_size > ceiling) + die(_("maximum input pack size cannot exceed %lu bytes " + "(half the effective pack.packSizeLimit)"), ceiling); + if (!max_input_pack_size) + max_input_pack_size = ceiling; + } + + keep_pack_list.cmp = fspathcmp; + string_list_sort(&keep_pack_list); + + packdir = mkpathdup("%s/pack", repo_get_object_directory(repo)); + + if (exclude_pack_file) + load_exclusions_from_file(exclude_pack_file, &pack_exclude); + if (exclude_loose_file) + load_exclusions_from_file(exclude_loose_file, &loose_exclude); + + sigchain_push(SIGTERM, term_handler); + sigchain_push(SIGHUP, term_handler); + sigchain_push(SIGINT, term_handler); + + do { + if (stop_signaled) + break; + trace2_region_enter("pack-aggregate", "cycle", repo); + ret = do_one_cycle(repo, packdir, &keep_pack_list, + &pack_exclude, &loose_exclude, &midx_exclude, + min_loose, min_packs, + max_loose_objects, max_objects, + max_packs, max_input_pack_size); + trace2_data_intmax("pack-aggregate", repo, "cycle-num", + ++cycle_count); + trace2_region_leave("pack-aggregate", "cycle", repo); + if (ret || once || stop_signaled) + break; + interruptible_sleep((unsigned int)interval); + } while (!stop_signaled); + + string_list_clear(&keep_pack_list, 0); + strset_clear(&pack_exclude); + strset_clear(&loose_exclude); + strset_clear(&midx_exclude); + free(packdir); + return ret ? 1 : 0; +} diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index af9390a46b9a69..63fb8788faa57a 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -44,6 +44,7 @@ #include "pack-mtimes.h" #include "parse-options.h" #include "pkt-line.h" +#include "path.h" #include "blob.h" #include "tree.h" #include "path-walk.h" @@ -208,10 +209,14 @@ static uint32_t write_layer; static int non_empty; static int reuse_delta = 1, reuse_object = 1; +static int prefer_reused_deltas; static int keep_unreachable, unpack_unreachable, include_tag; static timestamp_t unpack_unreachable_expiration; static int pack_loose_unreachable; static int cruft; +static int mark_bad_deltas; +static const char *emit_input_packs_path; +static const char *emit_input_loose_path; static int shallow = 0; static timestamp_t cruft_expiration; static int local; @@ -1470,6 +1475,28 @@ static void write_pack_file(void) &pack_idx_opts, hash, &idx_tmp_name); + if (mark_bad_deltas) { + struct strbuf marker_tmp = STRBUF_INIT; + size_t tmpname_len = tmpname.len; + int fd; + + fd = odb_mkstemp(the_repository->objects, + &marker_tmp, + "pack/tmp_baddeltas_XXXXXX"); + if (close(fd)) + die_errno(_("unable to close '%s'"), + marker_tmp.buf); + + strbuf_addstr(&tmpname, "baddeltas"); + if (finalize_object_file(the_repository, + marker_tmp.buf, + tmpname.buf)) + die(_("unable to rename temporary file to '%s'"), + tmpname.buf); + strbuf_release(&marker_tmp); + strbuf_setlen(&tmpname, tmpname_len); + } + if (write_bitmap_index) { size_t tmpname_len = tmpname.len; @@ -2827,9 +2854,15 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, * be considered, as even if we produce a suboptimal delta against * it, we will still save the transfer cost, as we already know * the other side has it and we won't send src_entry at all. + * + * If the source pack carries a ".baddeltas" marker, we treat its + * existing delta layout as untrusted: even if the two objects are + * in the same pack and neither is a delta, we have no reason to + * believe a previous packing run actually considered the pair. */ if (reuse_delta && IN_PACK(trg_entry) && IN_PACK(trg_entry) == IN_PACK(src_entry) && + !IN_PACK(trg_entry)->has_bad_deltas && !src_entry->preferred_base && trg_entry->in_pack_type != OBJ_REF_DELTA && trg_entry->in_pack_type != OBJ_OFS_DELTA) @@ -3804,6 +3837,108 @@ static int git_pack_config(const char *k, const char *v, static int stdin_packs_found_nr; static int stdin_packs_hints_nr; +/* + * Whether the --stdin-packs revision walk will actually run; used to skip + * seeding pending commits (and its expensive per-commit object lookups) when + * no walk is performed. Set in read_stdin_packs(). + */ +static int stdin_packs_need_walk; + +/* + * Return 1 if the object stored in pack `p` at byte offset `offset` is + * represented as a delta (OFS or REF), 0 otherwise. Only the object + * header is read, so this is cheap. + */ +static int pack_entry_is_delta(struct packed_git *p, off_t offset) +{ + struct pack_window *w_curs = NULL; + size_t avail, size; + enum object_type type; + unsigned char *buf; + int is_delta = 0; + + buf = use_pack(p, &w_curs, offset, &avail); + if (unpack_object_header_buffer(buf, avail, &type, &size)) + is_delta = (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA); + unuse_pack(&w_curs); + return is_delta; +} + +/* + * If the object stored in pack `p` at `offset` is a delta, resolve the + * object id of its base into `base_out` and return 1. Return 0 for a + * non-delta or on any parse failure. The cost is just an object-header + * parse, plus a revindex-backed offset lookup for OFS deltas. + */ +static int pack_entry_delta_base(struct packed_git *p, off_t offset, + struct object_id *base_out) +{ + struct pack_window *w_curs = NULL; + off_t curpos = offset; + size_t size; + enum object_type type; + int ret = 0; + + type = unpack_object_header(p, &w_curs, &curpos, &size); + if ((type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) && + !get_delta_base_oid(p, &w_curs, curpos, base_out, type, offset)) + ret = 1; + + unuse_pack(&w_curs); + return ret; +} + +/* + * When an object appears in more than one included pack, the first copy we + * saw (packs are visited newest-mtime first) is the one recorded in the + * packing list. If that copy is a plain base but another included pack + * stores the object as a delta, point the entry at the delta copy instead, + * so this run writes the object as a reused delta rather than as a base. + * Preferring the delta at this stage is safe -- if the chosen delta's base + * is not itself included, then check_object() will simply fall back and + * store this object as a base anyway. + */ +static void maybe_prefer_delta_copy(const struct object_id *oid, + struct packed_git *p, uint32_t pos) +{ + struct object_entry *entry, *base_entry; + struct object_id cand_base, base_of_base; + off_t ofs; + + if (!reuse_delta) + return; + entry = packlist_find(&to_pack, oid); + if (!entry || entry->preferred_base || !IN_PACK(entry)) + return; + + /* Only switch a plain base copy over to a delta copy. */ + if (pack_entry_is_delta(IN_PACK(entry), entry->in_pack_offset)) + return; + + ofs = nth_packed_object_offset(p, pos); + if (!pack_entry_delta_base(p, ofs, &cand_base)) + return; + + /* + * Switching would make `oid` a delta against `cand_base`. If + * `cand_base` is itself already recorded as a delta against `oid`, + * the switch forms a two-object cycle that a later + * break_delta_chains() pass would have to cut. Cutting it keeps one + * of the two deltas either way, so switching gains nothing here; skip + * it and leave `oid` a base, so the existing `cand_base -> oid` delta + * survives with no cycle to break. + */ + base_entry = packlist_find(&to_pack, &cand_base); + if (base_entry && !base_entry->preferred_base && IN_PACK(base_entry) && + pack_entry_delta_base(IN_PACK(base_entry), + base_entry->in_pack_offset, &base_of_base) && + oideq(&base_of_base, oid)) + return; + + oe_set_in_pack(&to_pack, entry, p); + entry->in_pack_offset = ofs; +} + static int add_object_entry_from_pack(const struct object_id *oid, struct packed_git *p, uint32_t pos, @@ -3815,8 +3950,11 @@ static int add_object_entry_from_pack(const struct object_id *oid, display_progress(progress_state, ++nr_seen); - if (have_duplicate_entry(oid, 0)) + if (have_duplicate_entry(oid, 0)) { + if (prefer_reused_deltas) + maybe_prefer_delta_copy(oid, p, pos); return 0; + } stdin_packs_found_nr++; @@ -3826,7 +3964,7 @@ static int add_object_entry_from_pack(const struct object_id *oid, if (packed_object_info(NULL, p, ofs, &oi) < 0) { die(_("could not get type of object %s in pack %s"), oid_to_hex(oid), p->pack_name); - } else if (type == OBJ_COMMIT) { + } else if (type == OBJ_COMMIT && stdin_packs_need_walk) { struct rev_info *revs = _data; /* * commits in included packs are used as starting points @@ -4095,6 +4233,7 @@ static void read_stdin_packs(struct repository *repo, { int prev_fetch_if_missing = repo->fetch_if_missing; struct rev_info revs; + int need_walk; /* * The revision walk may hit objects that are promised, only. As the @@ -4112,7 +4251,15 @@ static void read_stdin_packs(struct repository *repo, * That may cause us to avoid populating all of the namehash fields of * all included objects, but our goal is best-effort, since this is only * an optimization during delta selection. + * + * However, the walk is only needed for delta selection (which + * consumes the namehash) and for STDIN_PACKS_MODE_FOLLOW (which + * uses the walk to discover additional reachable objects); skip + * it when neither applies. */ + need_walk = (window && depth) || mode == STDIN_PACKS_MODE_FOLLOW; + stdin_packs_need_walk = need_walk; + revs.no_kept_objects = 1; revs.keep_pack_cache_flags |= KEPT_PACK_IN_CORE; revs.blob_objects = 1; @@ -4135,12 +4282,14 @@ static void read_stdin_packs(struct repository *repo, if (rev_list_unpacked) add_unreachable_loose_objects(&revs); - if (prepare_revision_walk(&revs)) - die(_("revision walk setup failed")); - traverse_commit_list(&revs, - show_commit_pack_hint, - show_object_pack_hint, - &mode); + if (need_walk) { + if (prepare_revision_walk(&revs)) + die(_("revision walk setup failed")); + traverse_commit_list(&revs, + show_commit_pack_hint, + show_object_pack_hint, + &mode); + } release_revisions(&revs); @@ -5120,6 +5269,68 @@ static int parse_stdin_packs_mode(const struct option *opt, const char *arg, return 0; } +static void emit_input_packs_to_file(const char *path) +{ + struct strbuf tmp = STRBUF_INIT; + struct packed_git *p; + FILE *fp; + + strbuf_addf(&tmp, "%s.tmp", path); + fp = fopen(tmp.buf, "w"); + if (!fp) + die_errno(_("unable to write '%s'"), tmp.buf); + /* + * This is deliberately a conservative snapshot of every local pack, + * not just packs selected by this invocation. A geometric repack can + * leave packs untouched but still include them in its replacement MIDX, + * so a concurrent aggregator must exclude them too. + */ + repo_for_each_pack(the_repository, p) { + /* Exclude alternates */ + if (!p->pack_local) + continue; + fprintf(fp, "%s\n", pack_basename(p)); + } + if (fclose(fp)) + die_errno(_("unable to write '%s'"), tmp.buf); + if (rename(tmp.buf, path)) + die_errno(_("unable to rename '%s' to '%s'"), tmp.buf, path); + strbuf_release(&tmp); +} + +static int emit_input_loose_cb(const struct object_id *oid, + const char *path UNUSED, + void *data) +{ + FILE *fp = data; + fprintf(fp, "%s\n", oid_to_hex(oid)); + return 0; +} + +static void emit_input_loose_to_file(const char *path) +{ + struct strbuf tmp = STRBUF_INIT; + FILE *fp; + + strbuf_addf(&tmp, "%s.tmp", path); + fp = fopen(tmp.buf, "w"); + if (!fp) + die_errno(_("unable to write '%s'"), tmp.buf); + /* + * Note: for_each_loose_file_in_source() walks only the local + * source (sources->next is skipped), thus excluding + * alternates and matching the "p->pack_local" check in + * emit_input_packs_to_file(). + */ + for_each_loose_file_in_source(the_repository->objects->sources, + emit_input_loose_cb, NULL, NULL, fp); + if (fclose(fp)) + die_errno(_("unable to write '%s'"), tmp.buf); + if (rename(tmp.buf, path)) + die_errno(_("unable to rename '%s' to '%s'"), tmp.buf, path); + strbuf_release(&tmp); +} + int cmd_pack_objects(int argc, const char **argv, const char *prefix, @@ -5164,6 +5375,9 @@ int cmd_pack_objects(int argc, N_("maximum length of delta chain allowed in the resulting pack")), OPT_BOOL(0, "reuse-delta", &reuse_delta, N_("reuse existing deltas")), + OPT_BOOL(0, "prefer-reused-deltas", &prefer_reused_deltas, + N_("when an object is in several included packs, " + "prefer a copy stored as a delta")), OPT_BOOL(0, "reuse-object", &reuse_object, N_("reuse existing objects")), OPT_BOOL(0, "delta-base-offset", &allow_ofs_delta, @@ -5201,6 +5415,8 @@ int cmd_pack_objects(int argc, N_("unpack unreachable objects newer than