summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJunio C Hamano <gitster@pobox.com>2022-08-03 20:36:06 (GMT)
committerJunio C Hamano <gitster@pobox.com>2022-08-03 20:36:06 (GMT)
commitacbec18d8e763069f55f895d15ebb37e9162357d (patch)
tree6b54a1fca554d1250d574ba79333d82961345a3a
parent350dc9f0e8974b6fcbdeb3808186c5a79c3e7386 (diff)
parent51d1b69a534b591aa4afdc6f64c0e7aa8d886961 (diff)
downloadgit-acbec18d8e763069f55f895d15ebb37e9162357d.zip
git-acbec18d8e763069f55f895d15ebb37e9162357d.tar.gz
git-acbec18d8e763069f55f895d15ebb37e9162357d.tar.bz2
Merge branch 'ds/midx-with-less-memory'
The codepath to write multi-pack index has been taught to release a large chunk of memory that holds an array of objects in the packs, as soon as it is done with the array, to reduce memory consumption. * ds/midx-with-less-memory: write_midx_bitmap(): drop unused refs_snapshot parameter midx: reduce memory pressure while writing bitmaps midx: extract bitmap write setup pack-bitmap-write: use const for hashes
-rw-r--r--midx.c72
-rw-r--r--pack-bitmap-write.c2
-rw-r--r--pack-bitmap.h2
3 files changed, 48 insertions, 28 deletions
diff --git a/midx.c b/midx.c
index 5f0dd38..4e956ca 100644
--- a/midx.c
+++ b/midx.c
@@ -1053,40 +1053,34 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr
return cb.commits;
}
-static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,
- struct write_midx_context *ctx,
- const char *refs_snapshot,
+static int write_midx_bitmap(const char *midx_name,
+ const unsigned char *midx_hash,
+ struct packing_data *pdata,
+ struct commit **commits,
+ uint32_t commits_nr,
+ uint32_t *pack_order,
unsigned flags)
{
- struct packing_data pdata;
- struct pack_idx_entry **index;
- struct commit **commits = NULL;
- uint32_t i, commits_nr;
+ int ret, i;
uint16_t options = 0;
- char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash));
- int ret;
-
- if (!ctx->entries_nr)
- BUG("cannot write a bitmap without any objects");
+ struct pack_idx_entry **index;
+ char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
+ hash_to_hex(midx_hash));
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
options |= BITMAP_OPT_HASH_CACHE;
- prepare_midx_packing_data(&pdata, ctx);
-
- commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, ctx);
-
/*
* Build the MIDX-order index based on pdata.objects (which is already
* in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
* this order).
*/
- ALLOC_ARRAY(index, pdata.nr_objects);
- for (i = 0; i < pdata.nr_objects; i++)
- index[i] = &pdata.objects[i].idx;
+ ALLOC_ARRAY(index, pdata->nr_objects);
+ for (i = 0; i < pdata->nr_objects; i++)
+ index[i] = &pdata->objects[i].idx;
bitmap_writer_show_progress(flags & MIDX_PROGRESS);
- bitmap_writer_build_type_index(&pdata, index, pdata.nr_objects);
+ bitmap_writer_build_type_index(pdata, index, pdata->nr_objects);
/*
* bitmap_writer_finish expects objects in lex order, but pack_order
@@ -1101,16 +1095,16 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,
* happens between bitmap_writer_build_type_index() and
* bitmap_writer_finish().
*/
- for (i = 0; i < pdata.nr_objects; i++)
- index[ctx->pack_order[i]] = &pdata.objects[i].idx;
+ for (i = 0; i < pdata->nr_objects; i++)
+ index[pack_order[i]] = &pdata->objects[i].idx;
bitmap_writer_select_commits(commits, commits_nr, -1);
- ret = bitmap_writer_build(&pdata);
+ ret = bitmap_writer_build(pdata);
if (ret < 0)
goto cleanup;
bitmap_writer_set_checksum(midx_hash);
- bitmap_writer_finish(index, pdata.nr_objects, bitmap_name, options);
+ bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options);
cleanup:
free(index);
@@ -1443,14 +1437,40 @@ static int write_midx_internal(const char *object_dir,
if (flags & MIDX_WRITE_REV_INDEX &&
git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
+
if (flags & MIDX_WRITE_BITMAP) {
- if (write_midx_bitmap(midx_name.buf, midx_hash, &ctx,
- refs_snapshot, flags) < 0) {
+ struct packing_data pdata;
+ struct commit **commits;
+ uint32_t commits_nr;
+
+ if (!ctx.entries_nr)
+ BUG("cannot write a bitmap without any objects");
+
+ prepare_midx_packing_data(&pdata, &ctx);
+
+ commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx);
+
+ /*
+ * The previous steps translated the information from
+ * 'entries' into information suitable for constructing
+ * bitmaps. We no longer need that array, so clear it to
+ * reduce memory pressure.
+ */
+ FREE_AND_NULL(ctx.entries);
+ ctx.entries_nr = 0;
+
+ if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata,
+ commits, commits_nr, ctx.pack_order,
+ flags) < 0) {
error(_("could not write multi-pack bitmap"));
result = 1;
goto cleanup;
}
}
+ /*
+ * NOTE: Do not use ctx.entries beyond this point, since it might
+ * have been freed in the previous if block.
+ */
if (ctx.m)
close_object_store(the_repository->objects);
diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c
index c43375b..4fcfaed 100644
--- a/pack-bitmap-write.c
+++ b/pack-bitmap-write.c
@@ -683,7 +683,7 @@ static void write_hash_cache(struct hashfile *f,
}
}
-void bitmap_writer_set_checksum(unsigned char *sha1)
+void bitmap_writer_set_checksum(const unsigned char *sha1)
{
hashcpy(writer.pack_checksum, sha1);
}
diff --git a/pack-bitmap.h b/pack-bitmap.h
index 3d3ddd7..f3a57ca 100644
--- a/pack-bitmap.h
+++ b/pack-bitmap.h
@@ -75,7 +75,7 @@ int bitmap_has_oid_in_uninteresting(struct bitmap_index *, const struct object_i
off_t get_disk_usage_from_bitmap(struct bitmap_index *, struct rev_info *);
void bitmap_writer_show_progress(int show);
-void bitmap_writer_set_checksum(unsigned char *sha1);
+void bitmap_writer_set_checksum(const unsigned char *sha1);
void bitmap_writer_build_type_index(struct packing_data *to_pack,
struct pack_idx_entry **index,
uint32_t index_nr);