From 957ba814bf93b698742ffa3cf37e4f665ed95b45 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 8 Sep 2021 08:29:30 +0000 Subject: commit-graph: when closing the graph, also release the slab The slab has information about the commit graph. That means that it is meaningless (and even misleading) when the commit graph was closed. This seems not to matter currently, but we're about to fix a Windows-specific bug where `git pull` does not close the object store before fetching (risking that an implicit auto-gc fails to remove the now-obsolete pack file(s)), and once we have that bug fix in place, it does matter: after that bug fix, we will open the object store, do some stuff with it, then close it, fetch, and then open it again, and do more stuff. If we close the commit graph without releasing the corresponding slab, we're hit by a symptom like this in t5520.19: BUG: commit-reach.c:85: bad generation skip 9223372036854775807 > 3 at 5cd378271655d43a3b4477520014f02213ad1546 Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 3860a0d..0998445 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -713,6 +713,7 @@ static void close_commit_graph_one(struct commit_graph *g) if (!g) return; + clear_commit_graph_data_slab(&commit_graph_data_slab); close_commit_graph_one(g->base_graph); free_commit_graph(g); } -- cgit v0.10.2-6-g49f6 From 7e44ff7a3983ad0c7be5c9edcfea2e8355ce9a65 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 8 Sep 2021 08:29:31 +0000 Subject: pull: release packs before fetching On Windows, files cannot be removed nor renamed if there are still handles held by a process. To remedy that, we try to release all open handles to any `.pack` file before e.g. repacking (which would want to remove the original `.pack` file(s) after it is done). Since the `read_cache_unmerged()` and/or the `get_oid()` call in `git pull` can cause `.pack` files to be opened, we need to release the open handles before calling `git fetch`: the latter process might want to spawn an auto-gc, which in turn might want to repack the objects. This commit is similar in spirit to 5bdece0d705 (gc/repack: release packs when needed, 2018-12-15). This fixes https://github.com/git-for-windows/git/issues/3336. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/builtin/pull.c b/builtin/pull.c index 3e13f81..d9f0156 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -26,6 +26,7 @@ #include "wt-status.h" #include "commit-reach.h" #include "sequencer.h" +#include "packfile.h" /** * Parses the value of --rebase. If value is a false value, returns @@ -998,6 +999,7 @@ int cmd_pull(int argc, const char **argv, const char *prefix) oidclr(&rebase_fork_point); } + close_object_store(the_repository->objects); if (run_fetch(repo, refspecs)) return 1; -- cgit v0.10.2-6-g49f6 From 3322a9d87f3b2121d2c62096f9261c8934c74056 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 9 Sep 2021 09:47:05 +0000 Subject: run-command: prettify the `RUN_COMMAND_*` flags The values were listed unaligned, and with powers of two spelled out in decimal. The list is easier to parse for human readers if the numbers are aligned and spelled out as powers of two (using the bit-shift operator `<<`). While at it, remove a code comment that was unclear at best, and confusing at worst. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/run-command.h b/run-command.h index af12967..3893193 100644 --- a/run-command.h +++ b/run-command.h @@ -233,13 +233,13 @@ int run_hook_ve(const char *const *env, const char *name, va_list args); */ int run_auto_maintenance(int quiet); -#define RUN_COMMAND_NO_STDIN 1 -#define RUN_GIT_CMD 2 /*If this is to be git sub-command */ -#define RUN_COMMAND_STDOUT_TO_STDERR 4 -#define RUN_SILENT_EXEC_FAILURE 8 -#define RUN_USING_SHELL 16 -#define RUN_CLEAN_ON_EXIT 32 -#define RUN_WAIT_AFTER_CLEAN 64 +#define RUN_COMMAND_NO_STDIN (1<<0) +#define RUN_GIT_CMD (1<<1) +#define RUN_COMMAND_STDOUT_TO_STDERR (1<<2) +#define RUN_SILENT_EXEC_FAILURE (1<<3) +#define RUN_USING_SHELL (1<<4) +#define RUN_CLEAN_ON_EXIT (1<<5) +#define RUN_WAIT_AFTER_CLEAN (1<<6) /** * Convenience functions that encapsulate a sequence of -- cgit v0.10.2-6-g49f6 From 28d04e1ec19777bf6382d016b6e624d0ff4336cd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 9 Sep 2021 09:47:06 +0000 Subject: run-command: offer to close the object store before running Especially on Windows, where files cannot be deleted if _any_ process holds an open file handle to them, it is important to close the object store (releasing all handles to all `.pack` files) before running a command that might spawn a garbage collection. This scenario is so common that we frequently see the pattern of closing the object store before running auto maintenance or another Git command. Let's make this much more convenient by teaching the `run_command()` machinery a new flag to release the object store before spawning the process. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/run-command.c b/run-command.c index f72e72c..e2dc624 100644 --- a/run-command.c +++ b/run-command.c @@ -8,6 +8,7 @@ #include "string-list.h" #include "quote.h" #include "config.h" +#include "packfile.h" void child_process_init(struct child_process *child) { @@ -740,6 +741,9 @@ fail_pipe: fflush(NULL); + if (cmd->close_object_store) + close_object_store(the_repository->objects); + #ifndef GIT_WINDOWS_NATIVE { int notify_pipe[2]; @@ -1044,6 +1048,7 @@ int run_command_v_opt_cd_env_tr2(const char **argv, int opt, const char *dir, cmd.use_shell = opt & RUN_USING_SHELL ? 1 : 0; cmd.clean_on_exit = opt & RUN_CLEAN_ON_EXIT ? 1 : 0; cmd.wait_after_clean = opt & RUN_WAIT_AFTER_CLEAN ? 1 : 0; + cmd.close_object_store = opt & RUN_CLOSE_OBJECT_STORE ? 1 : 0; cmd.dir = dir; cmd.env = env; cmd.trace2_child_class = tr2_class; diff --git a/run-command.h b/run-command.h index 3893193..ad207da 100644 --- a/run-command.h +++ b/run-command.h @@ -134,6 +134,14 @@ struct child_process { */ unsigned use_shell:1; + /** + * Release any open file handles to the object store before running + * the command; This is necessary e.g. when the spawned process may + * want to repack because that would delete `.pack` files (and on + * Windows, you cannot delete files that are still in use). + */ + unsigned close_object_store:1; + unsigned stdout_to_stderr:1; unsigned clean_on_exit:1; unsigned wait_after_clean:1; @@ -240,6 +248,7 @@ int run_auto_maintenance(int quiet); #define RUN_USING_SHELL (1<<4) #define RUN_CLEAN_ON_EXIT (1<<5) #define RUN_WAIT_AFTER_CLEAN (1<<6) +#define RUN_CLOSE_OBJECT_STORE (1<<7) /** * Convenience functions that encapsulate a sequence of -- cgit v0.10.2-6-g49f6 From 5a22a334cb757753230f1d73da36130513016830 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 9 Sep 2021 09:47:07 +0000 Subject: run_auto_maintenance(): implicitly close the object store Before spawning the auto maintenance, we need to make sure that we release all open file handles to all the `.pack` files (and MIDX files and commit-graph files and...) so that the maintenance process has the freedom to delete those files. So far, we did this manually every time before calling `run_auto_maintenance()`. With the new `close_object_store` flag, we can do that implicitly in that function, which is more robust because future callers won't be able to forget to close the object store. Note: this changes behavior slightly, as we previously _always_ closed the object store, but now we only close the object store when actually running the auto maintenance. In practice, this should not matter (if anything, it might speed up operations where auto maintenance is disabled). Suggested-by: Junio C Hamano Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/builtin/am.c b/builtin/am.c index 0c2ad96..f239e4d 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -1848,7 +1848,6 @@ next: */ if (!state->rebasing) { am_destroy(state); - close_object_store(the_repository->objects); run_auto_maintenance(state->quiet); } } diff --git a/builtin/fetch.c b/builtin/fetch.c index 25740c1..c9ac866 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -2133,8 +2133,6 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) NULL); } - close_object_store(the_repository->objects); - if (enable_auto_gc) run_auto_maintenance(verbosity < 0); diff --git a/builtin/merge.c b/builtin/merge.c index 22f2399..e4994e3 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -469,7 +469,6 @@ static void finish(struct commit *head_commit, * We ignore errors in 'gc --auto', since the * user should see them. */ - close_object_store(the_repository->objects); run_auto_maintenance(verbosity < 0); } } diff --git a/builtin/rebase.c b/builtin/rebase.c index 33e0961..ba09ebb 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -740,7 +740,6 @@ static int finish_rebase(struct rebase_options *opts) delete_ref(NULL, "REBASE_HEAD", NULL, REF_NO_DEREF); unlink(git_path_auto_merge(the_repository)); apply_autostash(state_dir_path("autostash", opts)); - close_object_store(the_repository->objects); /* * We ignore errors in 'git maintenance run --auto', since the * user should see them. diff --git a/run-command.c b/run-command.c index e2dc624..229bdff 100644 --- a/run-command.c +++ b/run-command.c @@ -1891,6 +1891,7 @@ int run_auto_maintenance(int quiet) return 0; maint.git_cmd = 1; + maint.close_object_store = 1; strvec_pushl(&maint.args, "maintenance", "run", "--auto", NULL); strvec_push(&maint.args, quiet ? "--quiet" : "--no-quiet"); -- cgit v0.10.2-6-g49f6 From c4dee2c0851f3a6b202afd2c9d979ed417f4bcdc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 9 Sep 2021 09:47:08 +0000 Subject: Close object store closer to spawning child processes In many cases where we spawned child processes that _may_ trigger a repack, we explicitly closed the object store first (so that the `repack` process can delete the `.pack` files, which would otherwise not be possible on Windows since files cannot be deleted as long as they as still in use). Wherever possible, we now use the new `close_object_store` bit of the `run_command()` API, to delay closing the object store even further. This makes the code easier to maintain because it is now more obvious that we only release those file handles because of those child processes. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/builtin/gc.c b/builtin/gc.c index f05d2f0..ddee9f8 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -663,8 +663,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix) gc_before_repack(); if (!repository_format_precious_objects) { - close_object_store(the_repository->objects); - if (run_command_v_opt(repack.v, RUN_GIT_CMD)) + if (run_command_v_opt(repack.v, + RUN_GIT_CMD | RUN_CLOSE_OBJECT_STORE)) die(FAILED_RUN, repack.v[0]); if (prune_expire) { @@ -848,7 +848,7 @@ static int run_write_commit_graph(struct maintenance_run_opts *opts) { struct child_process child = CHILD_PROCESS_INIT; - child.git_cmd = 1; + child.git_cmd = child.close_object_store = 1; strvec_pushl(&child.args, "commit-graph", "write", "--split", "--reachable", NULL); @@ -864,7 +864,6 @@ static int maintenance_task_commit_graph(struct maintenance_run_opts *opts) if (!the_repository->settings.core_commit_graph) return 0; - close_object_store(the_repository->objects); if (run_write_commit_graph(opts)) { error(_("failed to write commit-graph")); return 1; @@ -913,7 +912,7 @@ static int maintenance_task_gc(struct maintenance_run_opts *opts) { struct child_process child = CHILD_PROCESS_INIT; - child.git_cmd = 1; + child.git_cmd = child.close_object_store = 1; strvec_push(&child.args, "gc"); if (opts->auto_flag) @@ -923,7 +922,6 @@ static int maintenance_task_gc(struct maintenance_run_opts *opts) else strvec_push(&child.args, "--no-quiet"); - close_object_store(the_repository->objects); return run_command(&child); } @@ -1097,14 +1095,12 @@ static int multi_pack_index_expire(struct maintenance_run_opts *opts) { struct child_process child = CHILD_PROCESS_INIT; - child.git_cmd = 1; + child.git_cmd = child.close_object_store = 1; strvec_pushl(&child.args, "multi-pack-index", "expire", NULL); if (opts->quiet) strvec_push(&child.args, "--no-progress"); - close_object_store(the_repository->objects); - if (run_command(&child)) return error(_("'git multi-pack-index expire' failed")); @@ -1155,7 +1151,7 @@ static int multi_pack_index_repack(struct maintenance_run_opts *opts) { struct child_process child = CHILD_PROCESS_INIT; - child.git_cmd = 1; + child.git_cmd = child.close_object_store = 1; strvec_pushl(&child.args, "multi-pack-index", "repack", NULL); if (opts->quiet) @@ -1164,8 +1160,6 @@ static int multi_pack_index_repack(struct maintenance_run_opts *opts) strvec_pushf(&child.args, "--batch-size=%"PRIuMAX, (uintmax_t)get_auto_pack_size()); - close_object_store(the_repository->objects); - if (run_command(&child)) return error(_("'git multi-pack-index repack' failed")); diff --git a/builtin/pull.c b/builtin/pull.c index d9f0156..7513720 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -578,7 +578,7 @@ static int run_fetch(const char *repo, const char **refspecs) strvec_pushv(&args, refspecs); } else if (*refspecs) BUG("refspecs without repo?"); - ret = run_command_v_opt(args.v, RUN_GIT_CMD); + ret = run_command_v_opt(args.v, RUN_GIT_CMD | RUN_CLOSE_OBJECT_STORE); strvec_clear(&args); return ret; } @@ -999,7 +999,6 @@ int cmd_pull(int argc, const char **argv, const char *prefix) oidclr(&rebase_fork_point); } - close_object_store(the_repository->objects); if (run_fetch(repo, refspecs)) return 1; diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 2d1f97e..9d5e0e3 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -2580,10 +2580,9 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) proc.no_stdin = 1; proc.stdout_to_stderr = 1; proc.err = use_sideband ? -1 : 0; - proc.git_cmd = 1; + proc.git_cmd = proc.close_object_store = 1; proc.argv = argv_gc_auto; - close_object_store(the_repository->objects); if (!start_command(&proc)) { if (use_sideband) copy_to_sideband(proc.err, -1, NULL); -- cgit v0.10.2-6-g49f6