From c7944050af45c7384f97c712cb4d126672c7cfa6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 9 May 2019 07:22:31 -0700 Subject: commit-graph: fix the_repository reference The parse_commit_buffer() method takes a repository pointer, so it should not refer to the_repository anymore. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit.c b/commit.c index a5333c7..e4d1233 100644 --- a/commit.c +++ b/commit.c @@ -443,7 +443,7 @@ int parse_commit_buffer(struct repository *r, struct commit *item, const void *b item->date = parse_commit_date(bufptr, tail); if (check_graph) - load_commit_graph_info(the_repository, item); + load_commit_graph_info(r, item); return 0; } -- cgit v0.10.2-6-g49f6 From e103f7276f0d809c2935ebc1a3d68c6bbfaed23d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:37 -0700 Subject: commit-graph: return with errors during write The write_commit_graph() method uses die() to report failure and exit when confronted with an unexpected condition. This use of die() in a library function is incorrect and is now replaced by error() statements and an int return type. Return zero on success and a negative value on failure. Now that we use 'goto cleanup' to jump to the terminal condition on an error, we have new paths that could lead to uninitialized values. New initializers are added to correct for this. The builtins 'commit-graph', 'gc', and 'commit' call these methods, so update them to check the return value. Test that 'git commit-graph write' returns a proper error code when hitting a failure condition in write_commit_graph(). Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 537fdfd..2a1c4d7 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -141,6 +141,7 @@ static int graph_write(int argc, const char **argv) struct string_list *pack_indexes = NULL; struct string_list *commit_hex = NULL; struct string_list lines; + int result = 0; static struct option builtin_commit_graph_write_options[] = { OPT_STRING(0, "object-dir", &opts.obj_dir, @@ -168,10 +169,8 @@ static int graph_write(int argc, const char **argv) read_replace_refs = 0; - if (opts.reachable) { - write_commit_graph_reachable(opts.obj_dir, opts.append, 1); - return 0; - } + if (opts.reachable) + return write_commit_graph_reachable(opts.obj_dir, opts.append, 1); string_list_init(&lines, 0); if (opts.stdin_packs || opts.stdin_commits) { @@ -188,14 +187,15 @@ static int graph_write(int argc, const char **argv) UNLEAK(buf); } - write_commit_graph(opts.obj_dir, - pack_indexes, - commit_hex, - opts.append, - 1); + if (write_commit_graph(opts.obj_dir, + pack_indexes, + commit_hex, + opts.append, + 1)) + result = 1; UNLEAK(lines); - return 0; + return result; } int cmd_commit_graph(int argc, const char **argv, const char *prefix) diff --git a/builtin/commit.c b/builtin/commit.c index 2986553..b9ea722 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1669,8 +1669,9 @@ int cmd_commit(int argc, const char **argv, const char *prefix) "new_index file. Check that disk is not full and quota is\n" "not exceeded, and then \"git reset HEAD\" to recover.")); - if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) - write_commit_graph_reachable(get_object_directory(), 0, 0); + if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && + write_commit_graph_reachable(get_object_directory(), 0, 0)) + return 1; repo_rerere(the_repository, 0); run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); diff --git a/builtin/gc.c b/builtin/gc.c index 020f725..3984add 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -664,9 +664,10 @@ int cmd_gc(int argc, const char **argv, const char *prefix) clean_pack_garbage(); } - if (gc_write_commit_graph) - write_commit_graph_reachable(get_object_directory(), 0, - !quiet && !daemonized); + if (gc_write_commit_graph && + write_commit_graph_reachable(get_object_directory(), 0, + !quiet && !daemonized)) + return 1; if (auto_gc && too_many_loose_objects()) warning(_("There are too many unreachable loose objects; " diff --git a/commit-graph.c b/commit-graph.c index 66865ac..1b58d1d 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -851,27 +851,30 @@ static int add_ref_to_list(const char *refname, return 0; } -void write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress) +int write_commit_graph_reachable(const char *obj_dir, int append, + int report_progress) { struct string_list list = STRING_LIST_INIT_DUP; + int result; for_each_ref(add_ref_to_list, &list); - write_commit_graph(obj_dir, NULL, &list, append, report_progress); + result = write_commit_graph(obj_dir, NULL, &list, + append, report_progress); string_list_clear(&list, 0); + return result; } -void write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - int append, int report_progress) +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + int append, int report_progress) { struct packed_oid_list oids; struct packed_commit_list commits; struct hashfile *f; uint32_t i, count_distinct = 0; - char *graph_name; + char *graph_name = NULL; struct lock_file lk = LOCK_INIT; uint32_t chunk_ids[5]; uint64_t chunk_offsets[5]; @@ -883,15 +886,17 @@ void write_commit_graph(const char *obj_dir, uint64_t progress_cnt = 0; struct strbuf progress_title = STRBUF_INIT; unsigned long approx_nr_objects; + int res = 0; if (!commit_graph_compatible(the_repository)) - return; + return 0; oids.nr = 0; approx_nr_objects = approximate_object_count(); oids.alloc = approx_nr_objects / 32; oids.progress = NULL; oids.progress_done = 0; + commits.list = NULL; if (append) { prepare_commit_graph_one(the_repository, obj_dir); @@ -932,10 +937,16 @@ void write_commit_graph(const char *obj_dir, strbuf_setlen(&packname, dirlen); strbuf_addstr(&packname, pack_indexes->items[i].string); p = add_packed_git(packname.buf, packname.len, 1); - if (!p) - die(_("error adding pack %s"), packname.buf); - if (open_pack_index(p)) - die(_("error opening index for %s"), packname.buf); + if (!p) { + error(_("error adding pack %s"), packname.buf); + res = -1; + goto cleanup; + } + if (open_pack_index(p)) { + error(_("error opening index for %s"), packname.buf); + res = -1; + goto cleanup; + } for_each_object_in_pack(p, add_packed_commits, &oids, FOR_EACH_OBJECT_PACK_ORDER); close_pack(p); @@ -1006,8 +1017,11 @@ void write_commit_graph(const char *obj_dir, } stop_progress(&progress); - if (count_distinct >= GRAPH_EDGE_LAST_MASK) - die(_("the commit graph format cannot write %d commits"), count_distinct); + if (count_distinct >= GRAPH_EDGE_LAST_MASK) { + error(_("the commit graph format cannot write %d commits"), count_distinct); + res = -1; + goto cleanup; + } commits.nr = 0; commits.alloc = count_distinct; @@ -1039,16 +1053,21 @@ void write_commit_graph(const char *obj_dir, num_chunks = num_extra_edges ? 4 : 3; stop_progress(&progress); - if (commits.nr >= GRAPH_EDGE_LAST_MASK) - die(_("too many commits to write graph")); + if (commits.nr >= GRAPH_EDGE_LAST_MASK) { + error(_("too many commits to write graph")); + res = -1; + goto cleanup; + } compute_generation_numbers(&commits, report_progress); graph_name = get_commit_graph_filename(obj_dir); if (safe_create_leading_directories(graph_name)) { UNLEAK(graph_name); - die_errno(_("unable to create leading directories of %s"), - graph_name); + error(_("unable to create leading directories of %s"), + graph_name); + res = -1; + goto cleanup; } hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); @@ -1107,9 +1126,12 @@ void write_commit_graph(const char *obj_dir, finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); +cleanup: free(graph_name); free(commits.list); free(oids.list); + + return res; } #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 diff --git a/commit-graph.h b/commit-graph.h index 7dfb8c8..869717c 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -65,12 +65,18 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd, */ int generation_numbers_enabled(struct repository *r); -void write_commit_graph_reachable(const char *obj_dir, int append, +/* + * The write_commit_graph* methods return zero on success + * and a negative value on failure. Note that if the repository + * is not compatible with the commit-graph feature, then the + * methods will return 0 without writing a commit-graph. + */ +int write_commit_graph_reachable(const char *obj_dir, int append, int report_progress); -void write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - int append, int report_progress); +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + int append, int report_progress); int verify_commit_graph(struct repository *r, struct commit_graph *g); diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index e80c1ca..3b6fd0d 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -23,6 +23,14 @@ test_expect_success 'write graph with no packs' ' test_path_is_file info/commit-graph ' +test_expect_success 'close with correct error on bad input' ' + cd "$TRASH_DIRECTORY/full" && + echo doesnotexist >in && + { git commit-graph write --stdin-packs stderr; ret=$?; } && + test "$ret" = 1 && + test_i18ngrep "error adding pack" stderr +' + test_expect_success 'create commits and repack' ' cd "$TRASH_DIRECTORY/full" && for i in $(test_seq 3) -- cgit v0.10.2-6-g49f6 From 5af803945212af875670582ff153ee05ec368b83 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:38 -0700 Subject: commit-graph: collapse parameters into flags The write_commit_graph() and write_commit_graph_reachable() methods currently take two boolean parameters: 'append' and 'report_progress'. As we update these methods, adding more parameters this way becomes cluttered and hard to maintain. Collapse these parameters into a 'flags' parameter, and adjust the callers to provide flags as necessary. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 2a1c4d7..d8efa5b 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -142,6 +142,7 @@ static int graph_write(int argc, const char **argv) struct string_list *commit_hex = NULL; struct string_list lines; int result = 0; + unsigned int flags = COMMIT_GRAPH_PROGRESS; static struct option builtin_commit_graph_write_options[] = { OPT_STRING(0, "object-dir", &opts.obj_dir, @@ -166,11 +167,13 @@ static int graph_write(int argc, const char **argv) die(_("use at most one of --reachable, --stdin-commits, or --stdin-packs")); if (!opts.obj_dir) opts.obj_dir = get_object_directory(); + if (opts.append) + flags |= COMMIT_GRAPH_APPEND; read_replace_refs = 0; if (opts.reachable) - return write_commit_graph_reachable(opts.obj_dir, opts.append, 1); + return write_commit_graph_reachable(opts.obj_dir, flags); string_list_init(&lines, 0); if (opts.stdin_packs || opts.stdin_commits) { @@ -190,8 +193,7 @@ static int graph_write(int argc, const char **argv) if (write_commit_graph(opts.obj_dir, pack_indexes, commit_hex, - opts.append, - 1)) + flags)) result = 1; UNLEAK(lines); diff --git a/builtin/commit.c b/builtin/commit.c index b9ea722..b001ef5 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1670,7 +1670,7 @@ int cmd_commit(int argc, const char **argv, const char *prefix) "not exceeded, and then \"git reset HEAD\" to recover.")); if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && - write_commit_graph_reachable(get_object_directory(), 0, 0)) + write_commit_graph_reachable(get_object_directory(), 0)) return 1; repo_rerere(the_repository, 0); diff --git a/builtin/gc.c b/builtin/gc.c index 3984add..df2573f 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -665,8 +665,8 @@ int cmd_gc(int argc, const char **argv, const char *prefix) } if (gc_write_commit_graph && - write_commit_graph_reachable(get_object_directory(), 0, - !quiet && !daemonized)) + write_commit_graph_reachable(get_object_directory(), + !quiet && !daemonized ? COMMIT_GRAPH_PROGRESS : 0)) return 1; if (auto_gc && too_many_loose_objects()) diff --git a/commit-graph.c b/commit-graph.c index 1b58d1d..fc40b53 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -851,15 +851,14 @@ static int add_ref_to_list(const char *refname, return 0; } -int write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress) +int write_commit_graph_reachable(const char *obj_dir, unsigned int flags) { struct string_list list = STRING_LIST_INIT_DUP; int result; for_each_ref(add_ref_to_list, &list); result = write_commit_graph(obj_dir, NULL, &list, - append, report_progress); + flags); string_list_clear(&list, 0); return result; @@ -868,7 +867,7 @@ int write_commit_graph_reachable(const char *obj_dir, int append, int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, - int append, int report_progress) + unsigned int flags) { struct packed_oid_list oids; struct packed_commit_list commits; @@ -887,6 +886,8 @@ int write_commit_graph(const char *obj_dir, struct strbuf progress_title = STRBUF_INIT; unsigned long approx_nr_objects; int res = 0; + int append = flags & COMMIT_GRAPH_APPEND; + int report_progress = flags & COMMIT_GRAPH_PROGRESS; if (!commit_graph_compatible(the_repository)) return 0; diff --git a/commit-graph.h b/commit-graph.h index 869717c..01538b5 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -65,18 +65,20 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd, */ int generation_numbers_enabled(struct repository *r); +#define COMMIT_GRAPH_APPEND (1 << 0) +#define COMMIT_GRAPH_PROGRESS (1 << 1) + /* * The write_commit_graph* methods return zero on success * and a negative value on failure. Note that if the repository * is not compatible with the commit-graph feature, then the * methods will return 0 without writing a commit-graph. */ -int write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress); +int write_commit_graph_reachable(const char *obj_dir, unsigned int flags); int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, - int append, int report_progress); + unsigned int flags); int verify_commit_graph(struct repository *r, struct commit_graph *g); -- cgit v0.10.2-6-g49f6 From 10bd0be173b0ab9a9255fc15a768b2518d3eba57 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:39 -0700 Subject: commit-graph: remove Future Work section The commit-graph feature began with a long list of planned benefits, most of which are now complete. The future work section has only a few items left. As for making more algorithms aware of generation numbers, some are only waiting for generation number v2 to ensure the performance matches the existing behavior using commit date. It is unlikely that we will ever send a commit-graph file as part of the protocol, since we would need to verify the data, and that is expensive. If we want to start trusting remote content, then that item can be investigated again. While there is more work to be done on the feature, having a section of the docs devoted to a TODO list is wasteful and hard to keep up-to-date. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/Documentation/technical/commit-graph.txt b/Documentation/technical/commit-graph.txt index 7805b09..fb53341 100644 --- a/Documentation/technical/commit-graph.txt +++ b/Documentation/technical/commit-graph.txt @@ -127,23 +127,6 @@ Design Details helpful for these clones, anyway. The commit-graph will not be read or written when shallow commits are present. -Future Work ------------ - -- After computing and storing generation numbers, we must make graph - walks aware of generation numbers to gain the performance benefits they - enable. This will mostly be accomplished by swapping a commit-date-ordered - priority queue with one ordered by generation number. The following - operations are important candidates: - - - 'log --topo-order' - - 'tag --merged' - -- A server could provide a commit-graph file as part of the network protocol - to avoid extra calculations by clients. This feature is only of benefit if - the user is willing to trust the file, because verifying the file is correct - is as hard as computing it from scratch. - Related Links ------------- [0] https://bugs.chromium.org/p/git/issues/detail?id=8 -- cgit v0.10.2-6-g49f6 From c9905beade13efff6be9c15ebe03d07fe5278ccc Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:40 -0700 Subject: commit-graph: create write_commit_graph_context The write_commit_graph() method is too large and complex. To simplify it, we should extract several helper functions. However, we will risk repeating a lot of declarations related to progress incidators and object id or commit lists. Create a new write_commit_graph_context struct that contains the core data structures used in this process. Replace the other local variables with the values inside the context object. Following this change, we will start to lift code segments wholesale out of the write_commit_graph() method and into helper functions. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index fc40b53..6d7e83c 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -518,14 +518,38 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c); } +struct packed_commit_list { + struct commit **list; + int nr; + int alloc; +}; + +struct packed_oid_list { + struct object_id *list; + int nr; + int alloc; +}; + +struct write_commit_graph_context { + struct repository *r; + const char *obj_dir; + char *graph_name; + struct packed_oid_list oids; + struct packed_commit_list commits; + int num_extra_edges; + unsigned long approx_nr_objects; + struct progress *progress; + int progress_done; + uint64_t progress_cnt; + unsigned append:1, + report_progress:1; +}; + static void write_graph_chunk_fanout(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { int i, count = 0; - struct commit **list = commits; + struct commit **list = ctx->commits.list; /* * Write the first-level table (the list is sorted, @@ -533,10 +557,10 @@ static void write_graph_chunk_fanout(struct hashfile *f, * having to do eight extra binary search iterations). */ for (i = 0; i < 256; i++) { - while (count < nr_commits) { + while (count < ctx->commits.nr) { if ((*list)->object.oid.hash[0] != i) break; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); count++; list++; } @@ -546,14 +570,12 @@ static void write_graph_chunk_fanout(struct hashfile *f, } static void write_graph_chunk_oids(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; + struct commit **list = ctx->commits.list; int count; - for (count = 0; count < nr_commits; count++, list++) { - display_progress(progress, ++*progress_cnt); + for (count = 0; count < ctx->commits.nr; count++, list++) { + display_progress(ctx->progress, ++ctx->progress_cnt); hashwrite(f, (*list)->object.oid.hash, (int)hash_len); } } @@ -565,19 +587,17 @@ static const unsigned char *commit_to_sha1(size_t index, void *table) } static void write_graph_chunk_data(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; uint32_t num_extra_edges = 0; while (list < last) { struct commit_list *parent; int edge_value; uint32_t packedDate[2]; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); parse_commit_no_graph(*list); hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len); @@ -588,8 +608,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, edge_value = GRAPH_PARENT_NONE; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@ -609,8 +629,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) BUG("missing parent %s for commit %s", @@ -642,19 +662,16 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, } static void write_graph_chunk_extra_edges(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; struct commit_list *parent; while (list < last) { int num_parents = 0; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); for (parent = (*list)->parents; num_parents < 3 && parent; parent = parent->next) @@ -668,8 +685,8 @@ static void write_graph_chunk_extra_edges(struct hashfile *f, /* Since num_parents > 2, this initializer is safe. */ for (parent = (*list)->parents->next; parent; parent = parent->next) { int edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@ -693,125 +710,111 @@ static int commit_compare(const void *_a, const void *_b) return oidcmp(a, b); } -struct packed_commit_list { - struct commit **list; - int nr; - int alloc; -}; - -struct packed_oid_list { - struct object_id *list; - int nr; - int alloc; - struct progress *progress; - int progress_done; -}; - static int add_packed_commits(const struct object_id *oid, struct packed_git *pack, uint32_t pos, void *data) { - struct packed_oid_list *list = (struct packed_oid_list*)data; + struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data; enum object_type type; off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT; - if (list->progress) - display_progress(list->progress, ++list->progress_done); + if (ctx->progress) + display_progress(ctx->progress, ++ctx->progress_done); oi.typep = &type; - if (packed_object_info(the_repository, pack, offset, &oi) < 0) + if (packed_object_info(ctx->r, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid)); if (type != OBJ_COMMIT) return 0; - ALLOC_GROW(list->list, list->nr + 1, list->alloc); - oidcpy(&(list->list[list->nr]), oid); - list->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&(ctx->oids.list[ctx->oids.nr]), oid); + ctx->oids.nr++; return 0; } -static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit) +static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit) { struct commit_list *parent; for (parent = commit->parents; parent; parent = parent->next) { if (!(parent->item->object.flags & UNINTERESTING)) { - ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc); - oidcpy(&oids->list[oids->nr], &(parent->item->object.oid)); - oids->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(parent->item->object.oid)); + ctx->oids.nr++; parent->item->object.flags |= UNINTERESTING; } } } -static void close_reachable(struct packed_oid_list *oids, int report_progress) +static void close_reachable(struct write_commit_graph_context *ctx) { int i; struct commit *commit; - struct progress *progress = NULL; - if (report_progress) - progress = start_delayed_progress( - _("Loading known commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Loading known commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags |= UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); /* - * As this loop runs, oids->nr may grow, but not more + * As this loop runs, ctx->oids.nr may grow, but not more * than the number of missing commits in the reachable * closure. */ - if (report_progress) - progress = start_delayed_progress( - _("Expanding reachable commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Expanding reachable commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit && !parse_commit_no_graph(commit)) - add_missing_parents(oids, commit); + add_missing_parents(ctx, commit); } - stop_progress(&progress); + stop_progress(&ctx->progress); - if (report_progress) - progress = start_delayed_progress( - _("Clearing commit marks in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Clearing commit marks in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags &= ~UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); } -static void compute_generation_numbers(struct packed_commit_list* commits, - int report_progress) +static void compute_generation_numbers(struct write_commit_graph_context *ctx) { int i; struct commit_list *list = NULL; - struct progress *progress = NULL; - if (report_progress) - progress = start_progress( - _("Computing commit graph generation numbers"), - commits->nr); - for (i = 0; i < commits->nr; i++) { - display_progress(progress, i + 1); - if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY && - commits->list[i]->generation != GENERATION_NUMBER_ZERO) + if (ctx->report_progress) + ctx->progress = start_progress( + _("Computing commit graph generation numbers"), + ctx->commits.nr); + for (i = 0; i < ctx->commits.nr; i++) { + display_progress(ctx->progress, i + 1); + if (ctx->commits.list[i]->generation != GENERATION_NUMBER_INFINITY && + ctx->commits.list[i]->generation != GENERATION_NUMBER_ZERO) continue; - commit_list_insert(commits->list[i], &list); + commit_list_insert(ctx->commits.list[i], &list); while (list) { struct commit *current = list->item; struct commit_list *parent; @@ -838,7 +841,7 @@ static void compute_generation_numbers(struct packed_commit_list* commits, } } } - stop_progress(&progress); + stop_progress(&ctx->progress); } static int add_ref_to_list(const char *refname, @@ -869,8 +872,7 @@ int write_commit_graph(const char *obj_dir, struct string_list *commit_hex, unsigned int flags) { - struct packed_oid_list oids; - struct packed_commit_list commits; + struct write_commit_graph_context *ctx; struct hashfile *f; uint32_t i, count_distinct = 0; char *graph_name = NULL; @@ -878,44 +880,38 @@ int write_commit_graph(const char *obj_dir, uint32_t chunk_ids[5]; uint64_t chunk_offsets[5]; int num_chunks; - int num_extra_edges; struct commit_list *parent; - struct progress *progress = NULL; const unsigned hashsz = the_hash_algo->rawsz; - uint64_t progress_cnt = 0; struct strbuf progress_title = STRBUF_INIT; - unsigned long approx_nr_objects; int res = 0; - int append = flags & COMMIT_GRAPH_APPEND; - int report_progress = flags & COMMIT_GRAPH_PROGRESS; if (!commit_graph_compatible(the_repository)) return 0; - oids.nr = 0; - approx_nr_objects = approximate_object_count(); - oids.alloc = approx_nr_objects / 32; - oids.progress = NULL; - oids.progress_done = 0; - commits.list = NULL; - - if (append) { - prepare_commit_graph_one(the_repository, obj_dir); - if (the_repository->objects->commit_graph) - oids.alloc += the_repository->objects->commit_graph->num_commits; + ctx = xcalloc(1, sizeof(struct write_commit_graph_context)); + ctx->r = the_repository; + ctx->obj_dir = obj_dir; + ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0; + ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0; + + ctx->approx_nr_objects = approximate_object_count(); + ctx->oids.alloc = ctx->approx_nr_objects / 32; + + if (ctx->append) { + prepare_commit_graph_one(ctx->r, ctx->obj_dir); + if (ctx->r->objects->commit_graph) + ctx->oids.alloc += ctx->r->objects->commit_graph->num_commits; } - if (oids.alloc < 1024) - oids.alloc = 1024; - ALLOC_ARRAY(oids.list, oids.alloc); - - if (append && the_repository->objects->commit_graph) { - struct commit_graph *commit_graph = - the_repository->objects->commit_graph; - for (i = 0; i < commit_graph->num_commits; i++) { - const unsigned char *hash = commit_graph->chunk_oid_lookup + - commit_graph->hash_len * i; - hashcpy(oids.list[oids.nr++].hash, hash); + if (ctx->oids.alloc < 1024) + ctx->oids.alloc = 1024; + ALLOC_ARRAY(ctx->oids.list, ctx->oids.alloc); + + if (ctx->append && ctx->r->objects->commit_graph) { + struct commit_graph *g = ctx->r->objects->commit_graph; + for (i = 0; i < g->num_commits; i++) { + const unsigned char *hash = g->chunk_oid_lookup + g->hash_len * i; + hashcpy(ctx->oids.list[ctx->oids.nr++].hash, hash); } } @@ -924,14 +920,14 @@ int write_commit_graph(const char *obj_dir, int dirlen; strbuf_addf(&packname, "%s/pack/", obj_dir); dirlen = packname.len; - if (report_progress) { + if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Finding commits for commit graph in %d pack", "Finding commits for commit graph in %d packs", pack_indexes->nr), pack_indexes->nr); - oids.progress = start_delayed_progress(progress_title.buf, 0); - oids.progress_done = 0; + ctx->progress = start_delayed_progress(progress_title.buf, 0); + ctx->progress_done = 0; } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; @@ -948,75 +944,76 @@ int write_commit_graph(const char *obj_dir, res = -1; goto cleanup; } - for_each_object_in_pack(p, add_packed_commits, &oids, + for_each_object_in_pack(p, add_packed_commits, ctx, FOR_EACH_OBJECT_PACK_ORDER); close_pack(p); free(p); } - stop_progress(&oids.progress); + stop_progress(&ctx->progress); strbuf_reset(&progress_title); strbuf_release(&packname); } if (commit_hex) { - if (report_progress) { + if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Finding commits for commit graph from %d ref", "Finding commits for commit graph from %d refs", commit_hex->nr), commit_hex->nr); - progress = start_delayed_progress(progress_title.buf, - commit_hex->nr); + ctx->progress = start_delayed_progress( + progress_title.buf, + commit_hex->nr); } for (i = 0; i < commit_hex->nr; i++) { const char *end; struct object_id oid; struct commit *result; - display_progress(progress, i + 1); + display_progress(ctx->progress, i + 1); if (commit_hex->items[i].string && parse_oid_hex(commit_hex->items[i].string, &oid, &end)) continue; - result = lookup_commit_reference_gently(the_repository, &oid, 1); + result = lookup_commit_reference_gently(ctx->r, &oid, 1); if (result) { - ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc); - oidcpy(&oids.list[oids.nr], &(result->object.oid)); - oids.nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid)); + ctx->oids.nr++; } } - stop_progress(&progress); + stop_progress(&ctx->progress); strbuf_reset(&progress_title); } if (!pack_indexes && !commit_hex) { - if (report_progress) - oids.progress = start_delayed_progress( + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Finding commits for commit graph among packed objects"), - approx_nr_objects); - for_each_packed_object(add_packed_commits, &oids, + ctx->approx_nr_objects); + for_each_packed_object(add_packed_commits, ctx, FOR_EACH_OBJECT_PACK_ORDER); - if (oids.progress_done < approx_nr_objects) - display_progress(oids.progress, approx_nr_objects); - stop_progress(&oids.progress); + if (ctx->progress_done < ctx->approx_nr_objects) + display_progress(ctx->progress, ctx->approx_nr_objects); + stop_progress(&ctx->progress); } - close_reachable(&oids, report_progress); + close_reachable(ctx); - if (report_progress) - progress = start_delayed_progress( + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Counting distinct commits in commit graph"), - oids.nr); - display_progress(progress, 0); /* TODO: Measure QSORT() progress */ - QSORT(oids.list, oids.nr, commit_compare); + ctx->oids.nr); + display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */ + QSORT(ctx->oids.list, ctx->oids.nr, commit_compare); count_distinct = 1; - for (i = 1; i < oids.nr; i++) { - display_progress(progress, i + 1); - if (!oideq(&oids.list[i - 1], &oids.list[i])) + for (i = 1; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) count_distinct++; } - stop_progress(&progress); + stop_progress(&ctx->progress); if (count_distinct >= GRAPH_EDGE_LAST_MASK) { error(_("the commit graph format cannot write %d commits"), count_distinct); @@ -1024,54 +1021,54 @@ int write_commit_graph(const char *obj_dir, goto cleanup; } - commits.nr = 0; - commits.alloc = count_distinct; - ALLOC_ARRAY(commits.list, commits.alloc); + ctx->commits.alloc = count_distinct; + ALLOC_ARRAY(ctx->commits.list, ctx->commits.alloc); - num_extra_edges = 0; - if (report_progress) - progress = start_delayed_progress( + ctx->num_extra_edges = 0; + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Finding extra edges in commit graph"), - oids.nr); - for (i = 0; i < oids.nr; i++) { + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { int num_parents = 0; - display_progress(progress, i + 1); - if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i])) + display_progress(ctx->progress, i + 1); + if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) continue; - commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); - parse_commit_no_graph(commits.list[commits.nr]); + ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]); + parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]); - for (parent = commits.list[commits.nr]->parents; + for (parent = ctx->commits.list[ctx->commits.nr]->parents; parent; parent = parent->next) num_parents++; if (num_parents > 2) - num_extra_edges += num_parents - 1; + ctx->num_extra_edges += num_parents - 1; - commits.nr++; + ctx->commits.nr++; } - num_chunks = num_extra_edges ? 4 : 3; - stop_progress(&progress); + stop_progress(&ctx->progress); - if (commits.nr >= GRAPH_EDGE_LAST_MASK) { + if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) { error(_("too many commits to write graph")); res = -1; goto cleanup; } - compute_generation_numbers(&commits, report_progress); + compute_generation_numbers(ctx); - graph_name = get_commit_graph_filename(obj_dir); - if (safe_create_leading_directories(graph_name)) { - UNLEAK(graph_name); + num_chunks = ctx->num_extra_edges ? 4 : 3; + + ctx->graph_name = get_commit_graph_filename(ctx->obj_dir); + if (safe_create_leading_directories(ctx->graph_name)) { + UNLEAK(ctx->graph_name); error(_("unable to create leading directories of %s"), - graph_name); + ctx->graph_name); res = -1; goto cleanup; } - hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); hashwrite_be32(f, GRAPH_SIGNATURE); @@ -1084,7 +1081,7 @@ int write_commit_graph(const char *obj_dir, chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT; chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP; chunk_ids[2] = GRAPH_CHUNKID_DATA; - if (num_extra_edges) + if (ctx->num_extra_edges) chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES; else chunk_ids[3] = 0; @@ -1092,9 +1089,9 @@ int write_commit_graph(const char *obj_dir, chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; - chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr; - chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr; - chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges; + chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr; + chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr; + chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges; for (i = 0; i <= num_chunks; i++) { uint32_t chunk_write[3]; @@ -1105,32 +1102,33 @@ int write_commit_graph(const char *obj_dir, hashwrite(f, chunk_write, 12); } - if (report_progress) { + if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Writing out commit graph in %d pass", "Writing out commit graph in %d passes", num_chunks), num_chunks); - progress = start_delayed_progress( + ctx->progress = start_delayed_progress( progress_title.buf, - num_chunks * commits.nr); + num_chunks * ctx->commits.nr); } - write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - if (num_extra_edges) - write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt); - stop_progress(&progress); + write_graph_chunk_fanout(f, ctx); + write_graph_chunk_oids(f, hashsz, ctx); + write_graph_chunk_data(f, hashsz, ctx); + if (ctx->num_extra_edges) + write_graph_chunk_extra_edges(f, ctx); + stop_progress(&ctx->progress); strbuf_release(&progress_title); - close_commit_graph(the_repository); + close_commit_graph(ctx->r); finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); cleanup: free(graph_name); - free(commits.list); - free(oids.list); + free(ctx->commits.list); + free(ctx->oids.list); + free(ctx); return res; } -- cgit v0.10.2-6-g49f6 From ef5b83f2cfe11c2feb98b962881e9a8e6281e3ff Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:41 -0700 Subject: commit-graph: extract fill_oids_from_packs() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. This extracts fill_oids_from_packs() that reads the given pack-file list and fills the oid list in the context. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 6d7e83c..02e5f8c 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -867,6 +867,51 @@ int write_commit_graph_reachable(const char *obj_dir, unsigned int flags) return result; } +static int fill_oids_from_packs(struct write_commit_graph_context *ctx, + struct string_list *pack_indexes) +{ + uint32_t i; + struct strbuf progress_title = STRBUF_INIT; + struct strbuf packname = STRBUF_INIT; + int dirlen; + + strbuf_addf(&packname, "%s/pack/", ctx->obj_dir); + dirlen = packname.len; + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph in %d pack", + "Finding commits for commit graph in %d packs", + pack_indexes->nr), + pack_indexes->nr); + ctx->progress = start_delayed_progress(progress_title.buf, 0); + ctx->progress_done = 0; + } + for (i = 0; i < pack_indexes->nr; i++) { + struct packed_git *p; + strbuf_setlen(&packname, dirlen); + strbuf_addstr(&packname, pack_indexes->items[i].string); + p = add_packed_git(packname.buf, packname.len, 1); + if (!p) { + error(_("error adding pack %s"), packname.buf); + return -1; + } + if (open_pack_index(p)) { + error(_("error opening index for %s"), packname.buf); + return -1; + } + for_each_object_in_pack(p, add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + close_pack(p); + free(p); + } + + stop_progress(&ctx->progress); + strbuf_reset(&progress_title); + strbuf_release(&packname); + + return 0; +} + int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, @@ -916,42 +961,8 @@ int write_commit_graph(const char *obj_dir, } if (pack_indexes) { - struct strbuf packname = STRBUF_INIT; - int dirlen; - strbuf_addf(&packname, "%s/pack/", obj_dir); - dirlen = packname.len; - if (ctx->report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph in %d pack", - "Finding commits for commit graph in %d packs", - pack_indexes->nr), - pack_indexes->nr); - ctx->progress = start_delayed_progress(progress_title.buf, 0); - ctx->progress_done = 0; - } - for (i = 0; i < pack_indexes->nr; i++) { - struct packed_git *p; - strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(packname.buf, packname.len, 1); - if (!p) { - error(_("error adding pack %s"), packname.buf); - res = -1; - goto cleanup; - } - if (open_pack_index(p)) { - error(_("error opening index for %s"), packname.buf); - res = -1; - goto cleanup; - } - for_each_object_in_pack(p, add_packed_commits, ctx, - FOR_EACH_OBJECT_PACK_ORDER); - close_pack(p); - free(p); - } - stop_progress(&ctx->progress); - strbuf_reset(&progress_title); - strbuf_release(&packname); + if ((res = fill_oids_from_packs(ctx, pack_indexes))) + goto cleanup; } if (commit_hex) { -- cgit v0.10.2-6-g49f6 From 4c9efe850d75115a6deedd57072f1d7383bc03da Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:42 -0700 Subject: commit-graph: extract fill_oids_from_commit_hex() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. Extract fill_oids_from_commit_hex() that reads the given commit id list and fille the oid list in the context. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 02e5f8c..4fae1fc 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -912,6 +912,44 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, return 0; } +static void fill_oids_from_commit_hex(struct write_commit_graph_context *ctx, + struct string_list *commit_hex) +{ + uint32_t i; + struct strbuf progress_title = STRBUF_INIT; + + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph from %d ref", + "Finding commits for commit graph from %d refs", + commit_hex->nr), + commit_hex->nr); + ctx->progress = start_delayed_progress( + progress_title.buf, + commit_hex->nr); + } + for (i = 0; i < commit_hex->nr; i++) { + const char *end; + struct object_id oid; + struct commit *result; + + display_progress(ctx->progress, i + 1); + if (commit_hex->items[i].string && + parse_oid_hex(commit_hex->items[i].string, &oid, &end)) + continue; + + result = lookup_commit_reference_gently(ctx->r, &oid, 1); + + if (result) { + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid)); + ctx->oids.nr++; + } + } + stop_progress(&ctx->progress); + strbuf_release(&progress_title); +} + int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, @@ -965,38 +1003,8 @@ int write_commit_graph(const char *obj_dir, goto cleanup; } - if (commit_hex) { - if (ctx->report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph from %d ref", - "Finding commits for commit graph from %d refs", - commit_hex->nr), - commit_hex->nr); - ctx->progress = start_delayed_progress( - progress_title.buf, - commit_hex->nr); - } - for (i = 0; i < commit_hex->nr; i++) { - const char *end; - struct object_id oid; - struct commit *result; - - display_progress(ctx->progress, i + 1); - if (commit_hex->items[i].string && - parse_oid_hex(commit_hex->items[i].string, &oid, &end)) - continue; - - result = lookup_commit_reference_gently(ctx->r, &oid, 1); - - if (result) { - ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); - oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid)); - ctx->oids.nr++; - } - } - stop_progress(&ctx->progress); - strbuf_reset(&progress_title); - } + if (commit_hex) + fill_oids_from_commit_hex(ctx, commit_hex); if (!pack_indexes && !commit_hex) { if (ctx->report_progress) -- cgit v0.10.2-6-g49f6 From b2c83060525e69f628fb32836e311978e3899e6e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:42 -0700 Subject: commit-graph: extract fill_oids_from_all_packs() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. Extract fill_oids_from_all_packs() that reads all pack-files for commits and fills the oid list in the context. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 4fae1fc..61cb43d 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -950,6 +950,19 @@ static void fill_oids_from_commit_hex(struct write_commit_graph_context *ctx, strbuf_release(&progress_title); } +static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) +{ + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Finding commits for commit graph among packed objects"), + ctx->approx_nr_objects); + for_each_packed_object(add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + if (ctx->progress_done < ctx->approx_nr_objects) + display_progress(ctx->progress, ctx->approx_nr_objects); + stop_progress(&ctx->progress); +} + int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, @@ -1006,17 +1019,8 @@ int write_commit_graph(const char *obj_dir, if (commit_hex) fill_oids_from_commit_hex(ctx, commit_hex); - if (!pack_indexes && !commit_hex) { - if (ctx->report_progress) - ctx->progress = start_delayed_progress( - _("Finding commits for commit graph among packed objects"), - ctx->approx_nr_objects); - for_each_packed_object(add_packed_commits, ctx, - FOR_EACH_OBJECT_PACK_ORDER); - if (ctx->progress_done < ctx->approx_nr_objects) - display_progress(ctx->progress, ctx->approx_nr_objects); - stop_progress(&ctx->progress); - } + if (!pack_indexes && !commit_hex) + fill_oids_from_all_packs(ctx); close_reachable(ctx); -- cgit v0.10.2-6-g49f6 From 014e3440f524a620fd8aba92f1efeac737d9c23d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:43 -0700 Subject: commit-graph: extract count_distinct_commits() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. Extract count_distinct_commits(), which sorts the oids list, then iterates through to find duplicates. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 61cb43d..1a0a875 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -963,6 +963,27 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) stop_progress(&ctx->progress); } +static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i, count_distinct = 1; + + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Counting distinct commits in commit graph"), + ctx->oids.nr); + display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */ + QSORT(ctx->oids.list, ctx->oids.nr, commit_compare); + + for (i = 1; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) + count_distinct++; + } + stop_progress(&ctx->progress); + + return count_distinct; +} + int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, @@ -1024,19 +1045,7 @@ int write_commit_graph(const char *obj_dir, close_reachable(ctx); - if (ctx->report_progress) - ctx->progress = start_delayed_progress( - _("Counting distinct commits in commit graph"), - ctx->oids.nr); - display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */ - QSORT(ctx->oids.list, ctx->oids.nr, commit_compare); - count_distinct = 1; - for (i = 1; i < ctx->oids.nr; i++) { - display_progress(ctx->progress, i + 1); - if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) - count_distinct++; - } - stop_progress(&ctx->progress); + count_distinct = count_distinct_commits(ctx); if (count_distinct >= GRAPH_EDGE_LAST_MASK) { error(_("the commit graph format cannot write %d commits"), count_distinct); -- cgit v0.10.2-6-g49f6 From f998d542260ec643875e7f0fa860dbb43c2388ca Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:44 -0700 Subject: commit-graph: extract copy_oids_to_commits() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. Extract copy_oids_to_commits(), which fills the commits list with the distinct commits from the oids list. During this loop, it also counts the number of "extra" edges from octopus merges. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 1a0a875..72f9c5c 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -984,6 +984,37 @@ static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx) return count_distinct; } +static void copy_oids_to_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i; + struct commit_list *parent; + + ctx->num_extra_edges = 0; + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Finding extra edges in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + int num_parents = 0; + display_progress(ctx->progress, i + 1); + if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) + continue; + + ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]); + parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]); + + for (parent = ctx->commits.list[ctx->commits.nr]->parents; + parent; parent = parent->next) + num_parents++; + + if (num_parents > 2) + ctx->num_extra_edges += num_parents - 1; + + ctx->commits.nr++; + } + stop_progress(&ctx->progress); +} + int write_commit_graph(const char *obj_dir, struct string_list *pack_indexes, struct string_list *commit_hex, @@ -997,7 +1028,6 @@ int write_commit_graph(const char *obj_dir, uint32_t chunk_ids[5]; uint64_t chunk_offsets[5]; int num_chunks; - struct commit_list *parent; const unsigned hashsz = the_hash_algo->rawsz; struct strbuf progress_title = STRBUF_INIT; int res = 0; @@ -1056,30 +1086,7 @@ int write_commit_graph(const char *obj_dir, ctx->commits.alloc = count_distinct; ALLOC_ARRAY(ctx->commits.list, ctx->commits.alloc); - ctx->num_extra_edges = 0; - if (ctx->report_progress) - ctx->progress = start_delayed_progress( - _("Finding extra edges in commit graph"), - ctx->oids.nr); - for (i = 0; i < ctx->oids.nr; i++) { - int num_parents = 0; - display_progress(ctx->progress, i + 1); - if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) - continue; - - ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]); - parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]); - - for (parent = ctx->commits.list[ctx->commits.nr]->parents; - parent; parent = parent->next) - num_parents++; - - if (num_parents > 2) - ctx->num_extra_edges += num_parents - 1; - - ctx->commits.nr++; - } - stop_progress(&ctx->progress); + copy_oids_to_commits(ctx); if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) { error(_("too many commits to write graph")); -- cgit v0.10.2-6-g49f6 From 238def57fe7ba219c9e50094512fff1068ac5413 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 06:29:45 -0700 Subject: commit-graph: extract write_commit_graph_file() The write_commit_graph() method is too complex, so we are extracting helper functions one by one. Extract write_commit_graph_file() that takes all of the information in the context struct and writes the data to a commit-graph file. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/commit-graph.c b/commit-graph.c index 72f9c5c..9d2c72f 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1015,21 +1015,91 @@ static void copy_oids_to_commits(struct write_commit_graph_context *ctx) stop_progress(&ctx->progress); } -int write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - unsigned int flags) +static int write_commit_graph_file(struct write_commit_graph_context *ctx) { - struct write_commit_graph_context *ctx; + uint32_t i; struct hashfile *f; - uint32_t i, count_distinct = 0; - char *graph_name = NULL; struct lock_file lk = LOCK_INIT; uint32_t chunk_ids[5]; uint64_t chunk_offsets[5]; - int num_chunks; const unsigned hashsz = the_hash_algo->rawsz; struct strbuf progress_title = STRBUF_INIT; + int num_chunks = ctx->num_extra_edges ? 4 : 3; + + ctx->graph_name = get_commit_graph_filename(ctx->obj_dir); + if (safe_create_leading_directories(ctx->graph_name)) { + UNLEAK(ctx->graph_name); + error(_("unable to create leading directories of %s"), + ctx->graph_name); + return -1; + } + + hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR); + f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); + + hashwrite_be32(f, GRAPH_SIGNATURE); + + hashwrite_u8(f, GRAPH_VERSION); + hashwrite_u8(f, oid_version()); + hashwrite_u8(f, num_chunks); + hashwrite_u8(f, 0); /* unused padding byte */ + + chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT; + chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP; + chunk_ids[2] = GRAPH_CHUNKID_DATA; + if (ctx->num_extra_edges) + chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES; + else + chunk_ids[3] = 0; + chunk_ids[4] = 0; + + chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; + chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; + chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr; + chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr; + chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges; + + for (i = 0; i <= num_chunks; i++) { + uint32_t chunk_write[3]; + + chunk_write[0] = htonl(chunk_ids[i]); + chunk_write[1] = htonl(chunk_offsets[i] >> 32); + chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff); + hashwrite(f, chunk_write, 12); + } + + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Writing out commit graph in %d pass", + "Writing out commit graph in %d passes", + num_chunks), + num_chunks); + ctx->progress = start_delayed_progress( + progress_title.buf, + num_chunks * ctx->commits.nr); + } + write_graph_chunk_fanout(f, ctx); + write_graph_chunk_oids(f, hashsz, ctx); + write_graph_chunk_data(f, hashsz, ctx); + if (ctx->num_extra_edges) + write_graph_chunk_extra_edges(f, ctx); + stop_progress(&ctx->progress); + strbuf_release(&progress_title); + + close_commit_graph(ctx->r); + finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); + commit_lock_file(&lk); + + return 0; +} + +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + unsigned int flags) +{ + struct write_commit_graph_context *ctx; + uint32_t i, count_distinct = 0; int res = 0; if (!commit_graph_compatible(the_repository)) @@ -1096,75 +1166,10 @@ int write_commit_graph(const char *obj_dir, compute_generation_numbers(ctx); - num_chunks = ctx->num_extra_edges ? 4 : 3; - - ctx->graph_name = get_commit_graph_filename(ctx->obj_dir); - if (safe_create_leading_directories(ctx->graph_name)) { - UNLEAK(ctx->graph_name); - error(_("unable to create leading directories of %s"), - ctx->graph_name); - res = -1; - goto cleanup; - } - - hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR); - f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); - - hashwrite_be32(f, GRAPH_SIGNATURE); - - hashwrite_u8(f, GRAPH_VERSION); - hashwrite_u8(f, oid_version()); - hashwrite_u8(f, num_chunks); - hashwrite_u8(f, 0); /* unused padding byte */ - - chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT; - chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP; - chunk_ids[2] = GRAPH_CHUNKID_DATA; - if (ctx->num_extra_edges) - chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES; - else - chunk_ids[3] = 0; - chunk_ids[4] = 0; - - chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; - chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; - chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr; - chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr; - chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges; - - for (i = 0; i <= num_chunks; i++) { - uint32_t chunk_write[3]; - - chunk_write[0] = htonl(chunk_ids[i]); - chunk_write[1] = htonl(chunk_offsets[i] >> 32); - chunk_write[2] = htonl(chunk_offsets[i] & 0xffffffff); - hashwrite(f, chunk_write, 12); - } - - if (ctx->report_progress) { - strbuf_addf(&progress_title, - Q_("Writing out commit graph in %d pass", - "Writing out commit graph in %d passes", - num_chunks), - num_chunks); - ctx->progress = start_delayed_progress( - progress_title.buf, - num_chunks * ctx->commits.nr); - } - write_graph_chunk_fanout(f, ctx); - write_graph_chunk_oids(f, hashsz, ctx); - write_graph_chunk_data(f, hashsz, ctx); - if (ctx->num_extra_edges) - write_graph_chunk_extra_edges(f, ctx); - stop_progress(&ctx->progress); - strbuf_release(&progress_title); - - close_commit_graph(ctx->r); - finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); - commit_lock_file(&lk); + res = write_commit_graph_file(ctx); cleanup: - free(graph_name); + free(ctx->graph_name); free(ctx->commits.list); free(ctx->oids.list); free(ctx); -- cgit v0.10.2-6-g49f6