From 39c8df0cfe124565783bfc5eaaa3285c06b83fbf Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:43 +0200 Subject: packed-backend: don't adjust the reference count on lock/unlock The old code incremented the packed ref cache reference count when acquiring the packed-refs lock, and decremented the count when releasing the lock. This is unnecessary because: * Another process cannot change the packed-refs file because it is locked. * When we ourselves change the packed-refs file, we do so by first modifying the packed ref-cache, and then writing the data from the ref-cache to disk. So the packed ref-cache remains fresh because any changes that we plan to make to the file are made in the cache first anyway. So there is no reason for the cache to become stale. Moreover, the extra reference count causes a problem if we intentionally clear the packed refs cache, as we sometimes need to do if we change the cache in anticipation of writing a change to disk, but then the write to disk fails. In that case, `packed_refs_unlock()` would have no easy way to find the cache whose reference count it needs to decrement. This whole issue will soon become moot due to upcoming changes that avoid changing the in-memory cache as part of updating the packed-refs on disk, but this change makes that transition easier. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 412c850..b76f14e 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -525,7 +525,6 @@ int packed_refs_lock(struct ref_store *ref_store, int flags, struct strbuf *err) "packed_refs_lock"); static int timeout_configured = 0; static int timeout_value = 1000; - struct packed_ref_cache *packed_ref_cache; if (!timeout_configured) { git_config_get_int("core.packedrefstimeout", &timeout_value); @@ -560,9 +559,11 @@ int packed_refs_lock(struct ref_store *ref_store, int flags, struct strbuf *err) */ validate_packed_ref_cache(refs); - packed_ref_cache = get_packed_ref_cache(refs); - /* Increment the reference count to prevent it from being freed: */ - acquire_packed_ref_cache(packed_ref_cache); + /* + * Now make sure that the packed-refs file as it exists in the + * locked state is loaded into the cache: + */ + get_packed_ref_cache(refs); return 0; } @@ -576,7 +577,6 @@ void packed_refs_unlock(struct ref_store *ref_store) if (!is_lock_file_locked(&refs->lock)) die("BUG: packed_refs_unlock() called when not locked"); rollback_lock_file(&refs->lock); - release_packed_ref_cache(refs->cache); } int packed_refs_is_locked(struct ref_store *ref_store) -- cgit v0.10.2-6-g49f6 From 3bf4f56134c39a0c404ddf1a38779d67649e1c14 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:44 +0200 Subject: struct ref_transaction: add a place for backends to store data `packed_ref_store` is going to want to store some transaction-wide data, so make a place for it. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/refs-internal.h b/refs/refs-internal.h index b02dc5a..d7d344d 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -242,6 +242,7 @@ struct ref_transaction { size_t alloc; size_t nr; enum ref_transaction_state state; + void *backend_data; }; /* -- cgit v0.10.2-6-g49f6 From 2775d8724d73ba2705af72db075b0cdfba3c94f1 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:45 +0200 Subject: packed_ref_store: implement reference transactions Implement the methods needed to support reference transactions for the packed-refs backend. The new methods are not yet used. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/packed-backend.c b/refs/packed-backend.c index b76f14e..9ab65c5 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -748,25 +748,332 @@ static int packed_init_db(struct ref_store *ref_store, struct strbuf *err) return 0; } +/* + * Write the packed-refs from the cache to the packed-refs tempfile, + * incorporating any changes from `updates`. `updates` must be a + * sorted string list whose keys are the refnames and whose util + * values are `struct ref_update *`. On error, rollback the tempfile, + * write an error message to `err`, and return a nonzero value. + * + * The packfile must be locked before calling this function and will + * remain locked when it is done. + */ +static int write_with_updates(struct packed_ref_store *refs, + struct string_list *updates, + struct strbuf *err) +{ + struct ref_iterator *iter = NULL; + size_t i; + int ok; + FILE *out; + struct strbuf sb = STRBUF_INIT; + char *packed_refs_path; + + if (!is_lock_file_locked(&refs->lock)) + die("BUG: write_with_updates() called while unlocked"); + + /* + * If packed-refs is a symlink, we want to overwrite the + * symlinked-to file, not the symlink itself. Also, put the + * staging file next to it: + */ + packed_refs_path = get_locked_file_path(&refs->lock); + strbuf_addf(&sb, "%s.new", packed_refs_path); + free(packed_refs_path); + if (create_tempfile(&refs->tempfile, sb.buf) < 0) { + strbuf_addf(err, "unable to create file %s: %s", + sb.buf, strerror(errno)); + strbuf_release(&sb); + return -1; + } + strbuf_release(&sb); + + out = fdopen_tempfile(&refs->tempfile, "w"); + if (!out) { + strbuf_addf(err, "unable to fdopen packed-refs tempfile: %s", + strerror(errno)); + goto error; + } + + if (fprintf(out, "%s", PACKED_REFS_HEADER) < 0) + goto write_error; + + /* + * We iterate in parallel through the current list of refs and + * the list of updates, processing an entry from at least one + * of the lists each time through the loop. When the current + * list of refs is exhausted, set iter to NULL. When the list + * of updates is exhausted, leave i set to updates->nr. + */ + iter = packed_ref_iterator_begin(&refs->base, "", + DO_FOR_EACH_INCLUDE_BROKEN); + if ((ok = ref_iterator_advance(iter)) != ITER_OK) + iter = NULL; + + i = 0; + + while (iter || i < updates->nr) { + struct ref_update *update = NULL; + int cmp; + + if (i >= updates->nr) { + cmp = -1; + } else { + update = updates->items[i].util; + + if (!iter) + cmp = +1; + else + cmp = strcmp(iter->refname, update->refname); + } + + if (!cmp) { + /* + * There is both an old value and an update + * for this reference. Check the old value if + * necessary: + */ + if ((update->flags & REF_HAVE_OLD)) { + if (is_null_oid(&update->old_oid)) { + strbuf_addf(err, "cannot update ref '%s': " + "reference already exists", + update->refname); + goto error; + } else if (oidcmp(&update->old_oid, iter->oid)) { + strbuf_addf(err, "cannot update ref '%s': " + "is at %s but expected %s", + update->refname, + oid_to_hex(iter->oid), + oid_to_hex(&update->old_oid)); + goto error; + } + } + + /* Now figure out what to use for the new value: */ + if ((update->flags & REF_HAVE_NEW)) { + /* + * The update takes precedence. Skip + * the iterator over the unneeded + * value. + */ + if ((ok = ref_iterator_advance(iter)) != ITER_OK) + iter = NULL; + cmp = +1; + } else { + /* + * The update doesn't actually want to + * change anything. We're done with it. + */ + i++; + cmp = -1; + } + } else if (cmp > 0) { + /* + * There is no old value but there is an + * update for this reference. Make sure that + * the update didn't expect an existing value: + */ + if ((update->flags & REF_HAVE_OLD) && + !is_null_oid(&update->old_oid)) { + strbuf_addf(err, "cannot update ref '%s': " + "reference is missing but expected %s", + update->refname, + oid_to_hex(&update->old_oid)); + goto error; + } + } + + if (cmp < 0) { + /* Pass the old reference through. */ + + struct object_id peeled; + int peel_error = ref_iterator_peel(iter, &peeled); + + if (write_packed_entry(out, iter->refname, + iter->oid->hash, + peel_error ? NULL : peeled.hash)) + goto write_error; + + if ((ok = ref_iterator_advance(iter)) != ITER_OK) + iter = NULL; + } else if (is_null_oid(&update->new_oid)) { + /* + * The update wants to delete the reference, + * and the reference either didn't exist or we + * have already skipped it. So we're done with + * the update (and don't have to write + * anything). + */ + i++; + } else { + struct object_id peeled; + int peel_error = peel_object(update->new_oid.hash, + peeled.hash); + + if (write_packed_entry(out, update->refname, + update->new_oid.hash, + peel_error ? NULL : peeled.hash)) + goto write_error; + + i++; + } + } + + if (ok != ITER_DONE) { + strbuf_addf(err, "unable to write packed-refs file: " + "error iterating over old contents"); + goto error; + } + + if (close_tempfile(&refs->tempfile)) { + strbuf_addf(err, "error closing file %s: %s", + get_tempfile_path(&refs->tempfile), + strerror(errno)); + strbuf_release(&sb); + return -1; + } + + return 0; + +write_error: + strbuf_addf(err, "error writing to %s: %s", + get_tempfile_path(&refs->tempfile), strerror(errno)); + +error: + if (iter) + ref_iterator_abort(iter); + + delete_tempfile(&refs->tempfile); + return -1; +} + +struct packed_transaction_backend_data { + /* True iff the transaction owns the packed-refs lock. */ + int own_lock; + + struct string_list updates; +}; + +static void packed_transaction_cleanup(struct packed_ref_store *refs, + struct ref_transaction *transaction) +{ + struct packed_transaction_backend_data *data = transaction->backend_data; + + if (data) { + string_list_clear(&data->updates, 0); + + if (is_tempfile_active(&refs->tempfile)) + delete_tempfile(&refs->tempfile); + + if (data->own_lock && is_lock_file_locked(&refs->lock)) { + packed_refs_unlock(&refs->base); + data->own_lock = 0; + } + + free(data); + transaction->backend_data = NULL; + } + + transaction->state = REF_TRANSACTION_CLOSED; +} + static int packed_transaction_prepare(struct ref_store *ref_store, struct ref_transaction *transaction, struct strbuf *err) { - die("BUG: not implemented yet"); + struct packed_ref_store *refs = packed_downcast( + ref_store, + REF_STORE_READ | REF_STORE_WRITE | REF_STORE_ODB, + "ref_transaction_prepare"); + struct packed_transaction_backend_data *data; + size_t i; + int ret = TRANSACTION_GENERIC_ERROR; + + /* + * Note that we *don't* skip transactions with zero updates, + * because such a transaction might be executed for the side + * effect of ensuring that all of the references are peeled. + * If the caller wants to optimize away empty transactions, it + * should do so itself. + */ + + data = xcalloc(1, sizeof(*data)); + string_list_init(&data->updates, 0); + + transaction->backend_data = data; + + /* + * Stick the updates in a string list by refname so that we + * can sort them: + */ + for (i = 0; i < transaction->nr; i++) { + struct ref_update *update = transaction->updates[i]; + struct string_list_item *item = + string_list_append(&data->updates, update->refname); + + /* Store a pointer to update in item->util: */ + item->util = update; + } + string_list_sort(&data->updates); + + if (ref_update_reject_duplicates(&data->updates, err)) + goto failure; + + if (!is_lock_file_locked(&refs->lock)) { + if (packed_refs_lock(ref_store, 0, err)) + goto failure; + data->own_lock = 1; + } + + if (write_with_updates(refs, &data->updates, err)) + goto failure; + + transaction->state = REF_TRANSACTION_PREPARED; + return 0; + +failure: + packed_transaction_cleanup(refs, transaction); + return ret; } static int packed_transaction_abort(struct ref_store *ref_store, struct ref_transaction *transaction, struct strbuf *err) { - die("BUG: not implemented yet"); + struct packed_ref_store *refs = packed_downcast( + ref_store, + REF_STORE_READ | REF_STORE_WRITE | REF_STORE_ODB, + "ref_transaction_abort"); + + packed_transaction_cleanup(refs, transaction); + return 0; } static int packed_transaction_finish(struct ref_store *ref_store, struct ref_transaction *transaction, struct strbuf *err) { - die("BUG: not implemented yet"); + struct packed_ref_store *refs = packed_downcast( + ref_store, + REF_STORE_READ | REF_STORE_WRITE | REF_STORE_ODB, + "ref_transaction_finish"); + int ret = TRANSACTION_GENERIC_ERROR; + char *packed_refs_path; + + packed_refs_path = get_locked_file_path(&refs->lock); + if (rename_tempfile(&refs->tempfile, packed_refs_path)) { + strbuf_addf(err, "error replacing %s: %s", + refs->path, strerror(errno)); + goto cleanup; + } + + clear_packed_ref_cache(refs); + ret = 0; + +cleanup: + free(packed_refs_path); + packed_transaction_cleanup(refs, transaction); + return ret; } static int packed_initial_transaction_commit(struct ref_store *ref_store, diff --git a/refs/packed-backend.h b/refs/packed-backend.h index 03b7c1d..7af2897 100644 --- a/refs/packed-backend.h +++ b/refs/packed-backend.h @@ -1,6 +1,15 @@ #ifndef REFS_PACKED_BACKEND_H #define REFS_PACKED_BACKEND_H +/* + * Support for storing references in a `packed-refs` file. + * + * Note that this backend doesn't check for D/F conflicts, because it + * doesn't care about them. But usually it should be wrapped in a + * `files_ref_store` that prevents D/F conflicts from being created, + * even among packed refs. + */ + struct ref_store *packed_ref_store_create(const char *path, unsigned int store_flags); -- cgit v0.10.2-6-g49f6 From 2fb330ca7238088eea5c1926380feb187f4867bc Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:46 +0200 Subject: packed_delete_refs(): implement method Implement `packed_delete_refs()` using a reference transaction. This means that `files_delete_refs()` can use `refs_delete_refs()` instead of `repack_without_refs()` to delete any packed references, decreasing the coupling between the classes. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index fccbc24..2c78f63 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1157,7 +1157,7 @@ static int files_delete_refs(struct ref_store *ref_store, const char *msg, if (packed_refs_lock(refs->packed_ref_store, 0, &err)) goto error; - if (repack_without_refs(refs->packed_ref_store, refnames, &err)) { + if (refs_delete_refs(refs->packed_ref_store, msg, refnames, flags)) { packed_refs_unlock(refs->packed_ref_store); goto error; } diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 9ab65c5..9d5f76b 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -1086,7 +1086,50 @@ static int packed_initial_transaction_commit(struct ref_store *ref_store, static int packed_delete_refs(struct ref_store *ref_store, const char *msg, struct string_list *refnames, unsigned int flags) { - die("BUG: not implemented yet"); + struct packed_ref_store *refs = + packed_downcast(ref_store, REF_STORE_WRITE, "delete_refs"); + struct strbuf err = STRBUF_INIT; + struct ref_transaction *transaction; + struct string_list_item *item; + int ret; + + (void)refs; /* We need the check above, but don't use the variable */ + + if (!refnames->nr) + return 0; + + /* + * Since we don't check the references' old_oids, the + * individual updates can't fail, so we can pack all of the + * updates into a single transaction. + */ + + transaction = ref_store_transaction_begin(ref_store, &err); + if (!transaction) + return -1; + + for_each_string_list_item(item, refnames) { + if (ref_transaction_delete(transaction, item->string, NULL, + flags, msg, &err)) { + warning(_("could not delete reference %s: %s"), + item->string, err.buf); + strbuf_reset(&err); + } + } + + ret = ref_transaction_commit(transaction, &err); + + if (ret) { + if (refnames->nr == 1) + error(_("could not delete reference %s: %s"), + refnames->items[0].string, err.buf); + else + error(_("could not delete references: %s"), err.buf); + } + + ref_transaction_free(transaction); + strbuf_release(&err); + return ret; } static int packed_pack_refs(struct ref_store *ref_store, unsigned int flags) -- cgit v0.10.2-6-g49f6 From 27d03d04d5b7e86a260d7afa283c9905f64de350 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:47 +0200 Subject: files_pack_refs(): use a reference transaction to write packed refs Now that the packed reference store supports transactions, we can use a transaction to write the packed versions of references that we want to pack. This decreases the coupling between `files_ref_store` and `packed_ref_store`. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index 2c78f63..3475c6f 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1100,6 +1100,11 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) int ok; struct ref_to_prune *refs_to_prune = NULL; struct strbuf err = STRBUF_INIT; + struct ref_transaction *transaction; + + transaction = ref_store_transaction_begin(refs->packed_ref_store, &err); + if (!transaction) + return -1; packed_refs_lock(refs->packed_ref_store, LOCK_DIE_ON_ERROR, &err); @@ -1115,12 +1120,14 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) continue; /* - * Create an entry in the packed-refs cache equivalent - * to the one from the loose ref cache, except that - * we don't copy the peeled status, because we want it - * to be re-peeled. + * Add a reference creation for this reference to the + * packed-refs transaction: */ - add_packed_ref(refs->packed_ref_store, iter->refname, iter->oid); + if (ref_transaction_update(transaction, iter->refname, + iter->oid->hash, NULL, + REF_NODEREF, NULL, &err)) + die("failure preparing to create packed reference %s: %s", + iter->refname, err.buf); /* Schedule the loose reference for pruning if requested. */ if ((flags & PACK_REFS_PRUNE)) { @@ -1134,8 +1141,11 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) if (ok != ITER_DONE) die("error while iterating over references"); - if (commit_packed_refs(refs->packed_ref_store, &err)) - die("unable to overwrite old ref-pack file: %s", err.buf); + if (ref_transaction_commit(transaction, &err)) + die("unable to write new packed-refs: %s", err.buf); + + ref_transaction_free(transaction); + packed_refs_unlock(refs->packed_ref_store); prune_refs(refs, refs_to_prune); -- cgit v0.10.2-6-g49f6 From 22b09cdfadf7a48f6503fddf51082c66541cf1d6 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:48 +0200 Subject: prune_refs(): also free the linked list At least since v1.7, the elements of the `refs_to_prune` linked list have been leaked. Fix the leak by teaching `prune_refs()` to free the list elements as it processes them. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index 3475c6f..60031fe 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1057,11 +1057,17 @@ static void prune_ref(struct files_ref_store *refs, struct ref_to_prune *r) strbuf_release(&err); } -static void prune_refs(struct files_ref_store *refs, struct ref_to_prune *r) +/* + * Prune the loose versions of the references in the linked list + * `*refs_to_prune`, freeing the entries in the list as we go. + */ +static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_to_prune) { - while (r) { + while (*refs_to_prune) { + struct ref_to_prune *r = *refs_to_prune; + *refs_to_prune = r->next; prune_ref(refs, r); - r = r->next; + free(r); } } @@ -1148,7 +1154,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) packed_refs_unlock(refs->packed_ref_store); - prune_refs(refs, refs_to_prune); + prune_refs(refs, &refs_to_prune); strbuf_release(&err); return 0; } -- cgit v0.10.2-6-g49f6 From 1444bfe0271a1dd46283a298e1a43c6565c38271 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:49 +0200 Subject: files_initial_transaction_commit(): use a transaction for packed refs Use a `packed_ref_store` transaction in the implementation of `files_initial_transaction_commit()` rather than using internal features of the packed ref store. This further decouples `files_ref_store` from `packed_ref_store`. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index 60031fe..2700e3b 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2669,6 +2669,7 @@ static int files_initial_transaction_commit(struct ref_store *ref_store, size_t i; int ret = 0; struct string_list affected_refnames = STRING_LIST_INIT_NODUP; + struct ref_transaction *packed_transaction = NULL; assert(err); @@ -2701,6 +2702,12 @@ static int files_initial_transaction_commit(struct ref_store *ref_store, &affected_refnames)) die("BUG: initial ref transaction called with existing refs"); + packed_transaction = ref_store_transaction_begin(refs->packed_ref_store, err); + if (!packed_transaction) { + ret = TRANSACTION_GENERIC_ERROR; + goto cleanup; + } + for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; @@ -2713,6 +2720,15 @@ static int files_initial_transaction_commit(struct ref_store *ref_store, ret = TRANSACTION_NAME_CONFLICT; goto cleanup; } + + /* + * Add a reference creation for this reference to the + * packed-refs transaction: + */ + ref_transaction_add_update(packed_transaction, update->refname, + update->flags & ~REF_HAVE_OLD, + update->new_oid.hash, update->old_oid.hash, + NULL); } if (packed_refs_lock(refs->packed_ref_store, 0, err)) { @@ -2720,21 +2736,14 @@ static int files_initial_transaction_commit(struct ref_store *ref_store, goto cleanup; } - for (i = 0; i < transaction->nr; i++) { - struct ref_update *update = transaction->updates[i]; - - if ((update->flags & REF_HAVE_NEW) && - !is_null_oid(&update->new_oid)) - add_packed_ref(refs->packed_ref_store, update->refname, - &update->new_oid); - } - - if (commit_packed_refs(refs->packed_ref_store, err)) { + if (initial_ref_transaction_commit(packed_transaction, err)) { ret = TRANSACTION_GENERIC_ERROR; goto cleanup; } cleanup: + if (packed_transaction) + ref_transaction_free(packed_transaction); packed_refs_unlock(refs->packed_ref_store); transaction->state = REF_TRANSACTION_CLOSED; string_list_clear(&affected_refnames, 0); -- cgit v0.10.2-6-g49f6 From 6a2a7736d867dac39f2c54833fd0a65107c7cc5b Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:50 +0200 Subject: t1404: demonstrate two problems with reference transactions Currently, a loose reference is deleted even before locking the `packed-refs` file, let alone deleting any packed version of the reference. This leads to two problems, demonstrated by two new tests: * While a reference is being deleted, other processes might see the old, packed value of the reference for a moment before the packed version is deleted. Normally this would be hard to observe, but we can prolong the window by locking the `packed-refs` file externally before running `update-ref`, then unlocking it before `update-ref`'s attempt to acquire the lock times out. * If the `packed-refs` file is locked so long that `update-ref` fails to lock it, then the reference can be left permanently in the incorrect state described in the previous point. In a moment, both problems will be fixed. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/t/t1404-update-ref-errors.sh b/t/t1404-update-ref-errors.sh index c34ece4..64a8134 100755 --- a/t/t1404-update-ref-errors.sh +++ b/t/t1404-update-ref-errors.sh @@ -404,4 +404,77 @@ test_expect_success 'broken reference blocks indirect create' ' test_cmp expected output.err ' +test_expect_failure 'no bogus intermediate values during delete' ' + prefix=refs/slow-transaction && + # Set up a reference with differing loose and packed versions: + git update-ref $prefix/foo $C && + git pack-refs --all && + git update-ref $prefix/foo $D && + git for-each-ref $prefix >unchanged && + # Now try to update the reference, but hold the `packed-refs` lock + # for a while to see what happens while the process is blocked: + : >.git/packed-refs.lock && + test_when_finished "rm -f .git/packed-refs.lock" && + { + # Note: the following command is intentionally run in the + # background. We increase the timeout so that `update-ref` + # attempts to acquire the `packed-refs` lock for longer than + # it takes for us to do the check then delete it: + git -c core.packedrefstimeout=3000 update-ref -d $prefix/foo & + } && + pid2=$! && + # Give update-ref plenty of time to get to the point where it tries + # to lock packed-refs: + sleep 1 && + # Make sure that update-ref did not complete despite the lock: + kill -0 $pid2 && + # Verify that the reference still has its old value: + sha1=$(git rev-parse --verify --quiet $prefix/foo || echo undefined) && + case "$sha1" in + $D) + # This is what we hope for; it means that nothing + # user-visible has changed yet. + : ;; + undefined) + # This is not correct; it means the deletion has happened + # already even though update-ref should not have been + # able to acquire the lock yet. + echo "$prefix/foo deleted prematurely" && + break + ;; + $C) + # This value should never be seen. Probably the loose + # reference has been deleted but the packed reference + # is still there: + echo "$prefix/foo incorrectly observed to be C" && + break + ;; + *) + # WTF? + echo "unexpected value observed for $prefix/foo: $sha1" && + break + ;; + esac >out && + rm -f .git/packed-refs.lock && + wait $pid2 && + test_must_be_empty out && + test_must_fail git rev-parse --verify --quiet $prefix/foo +' + +test_expect_failure 'delete fails cleanly if packed-refs file is locked' ' + prefix=refs/locked-packed-refs && + # Set up a reference with differing loose and packed versions: + git update-ref $prefix/foo $C && + git pack-refs --all && + git update-ref $prefix/foo $D && + git for-each-ref $prefix >unchanged && + # Now try to delete it while the `packed-refs` lock is held: + : >.git/packed-refs.lock && + test_when_finished "rm -f .git/packed-refs.lock" && + test_must_fail git update-ref -d $prefix/foo >out 2>err && + git for-each-ref $prefix >actual && + test_i18ngrep "Unable to create $Q.*packed-refs.lock$Q: File exists" err && + test_cmp unchanged actual +' + test_done -- cgit v0.10.2-6-g49f6 From dc39e099422b1d44f6230f557f94f7945c7521a7 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:51 +0200 Subject: files_ref_store: use a transaction to update packed refs When processing a `files_ref_store` transaction, it is sometimes necessary to delete some references from the "packed-refs" file. Do that using a reference transaction conducted against the `packed_ref_store`. This change further decouples `files_ref_store` from `packed_ref_store`. It also fixes multiple problems, including the two revealed by test cases added in the previous commit. First, the old code didn't obtain the `packed-refs` lock until `files_transaction_finish()`. This means that a failure to acquire the `packed-refs` lock (e.g., due to contention with another process) wasn't detected until it was too late (problems like this are supposed to be detected in the "prepare" phase). The new code acquires the `packed-refs` lock in `files_transaction_prepare()`, the same stage of the processing when the loose reference locks are being acquired, removing another reason why the "prepare" phase might succeed and the "finish" phase might nevertheless fail. Second, the old code deleted the loose version of a reference before deleting any packed version of the same reference. This left a moment when another process might think that the packed version of the reference is current, which is incorrect. (Even worse, the packed version of the reference can be arbitrarily old, and might even point at an object that has since been garbage-collected.) Third, if a reference deletion fails to acquire the `packed-refs` lock altogether, then the old code might leave the repository in the incorrect state (possibly corrupt) described in the previous paragraph. Now we activate the new "packed-refs" file (sans any references that are being deleted) *before* deleting the corresponding loose references. But we hold the "packed-refs" lock until after the loose references have been finalized, thus preventing a simultaneous "pack-refs" process from packing the loose version of the reference in the time gap, which would otherwise defeat our attempt to delete it. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index 2700e3b..29eb5e8 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2397,13 +2397,22 @@ static int lock_ref_for_update(struct files_ref_store *refs, return 0; } +struct files_transaction_backend_data { + struct ref_transaction *packed_transaction; + int packed_refs_locked; +}; + /* * Unlock any references in `transaction` that are still locked, and * mark the transaction closed. */ -static void files_transaction_cleanup(struct ref_transaction *transaction) +static void files_transaction_cleanup(struct files_ref_store *refs, + struct ref_transaction *transaction) { size_t i; + struct files_transaction_backend_data *backend_data = + transaction->backend_data; + struct strbuf err = STRBUF_INIT; for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; @@ -2415,6 +2424,17 @@ static void files_transaction_cleanup(struct ref_transaction *transaction) } } + if (backend_data->packed_transaction && + ref_transaction_abort(backend_data->packed_transaction, &err)) { + error("error aborting transaction: %s", err.buf); + strbuf_release(&err); + } + + if (backend_data->packed_refs_locked) + packed_refs_unlock(refs->packed_ref_store); + + free(backend_data); + transaction->state = REF_TRANSACTION_CLOSED; } @@ -2431,12 +2451,17 @@ static int files_transaction_prepare(struct ref_store *ref_store, char *head_ref = NULL; int head_type; struct object_id head_oid; + struct files_transaction_backend_data *backend_data; + struct ref_transaction *packed_transaction = NULL; assert(err); if (!transaction->nr) goto cleanup; + backend_data = xcalloc(1, sizeof(*backend_data)); + transaction->backend_data = backend_data; + /* * Fail if a refname appears more than once in the * transaction. (If we end up splitting up any updates using @@ -2503,6 +2528,41 @@ static int files_transaction_prepare(struct ref_store *ref_store, head_ref, &affected_refnames, err); if (ret) break; + + if (update->flags & REF_DELETING && + !(update->flags & REF_LOG_ONLY) && + !(update->flags & REF_ISPRUNING)) { + /* + * This reference has to be deleted from + * packed-refs if it exists there. + */ + if (!packed_transaction) { + packed_transaction = ref_store_transaction_begin( + refs->packed_ref_store, err); + if (!packed_transaction) { + ret = TRANSACTION_GENERIC_ERROR; + goto cleanup; + } + + backend_data->packed_transaction = + packed_transaction; + } + + ref_transaction_add_update( + packed_transaction, update->refname, + update->flags & ~REF_HAVE_OLD, + update->new_oid.hash, update->old_oid.hash, + NULL); + } + } + + if (packed_transaction) { + if (packed_refs_lock(refs->packed_ref_store, 0, err)) { + ret = TRANSACTION_GENERIC_ERROR; + goto cleanup; + } + backend_data->packed_refs_locked = 1; + ret = ref_transaction_prepare(packed_transaction, err); } cleanup: @@ -2510,7 +2570,7 @@ cleanup: string_list_clear(&affected_refnames, 0); if (ret) - files_transaction_cleanup(transaction); + files_transaction_cleanup(refs, transaction); else transaction->state = REF_TRANSACTION_PREPARED; @@ -2525,9 +2585,10 @@ static int files_transaction_finish(struct ref_store *ref_store, files_downcast(ref_store, 0, "ref_transaction_finish"); size_t i; int ret = 0; - struct string_list refs_to_delete = STRING_LIST_INIT_NODUP; - struct string_list_item *ref_to_delete; struct strbuf sb = STRBUF_INIT; + struct files_transaction_backend_data *backend_data; + struct ref_transaction *packed_transaction; + assert(err); @@ -2536,6 +2597,9 @@ static int files_transaction_finish(struct ref_store *ref_store, return 0; } + backend_data = transaction->backend_data; + packed_transaction = backend_data->packed_transaction; + /* Perform updates first so live commits remain referenced */ for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; @@ -2571,7 +2635,23 @@ static int files_transaction_finish(struct ref_store *ref_store, } } } - /* Perform deletes now that updates are safely completed */ + + /* + * Perform deletes now that updates are safely completed. + * + * First delete any packed versions of the references, while + * retaining the packed-refs lock: + */ + if (packed_transaction) { + ret = ref_transaction_commit(packed_transaction, err); + ref_transaction_free(packed_transaction); + packed_transaction = NULL; + backend_data->packed_transaction = NULL; + if (ret) + goto cleanup; + } + + /* Now delete the loose versions of the references: */ for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; struct ref_lock *lock = update->backend_data; @@ -2589,39 +2669,27 @@ static int files_transaction_finish(struct ref_store *ref_store, } update->flags |= REF_DELETED_LOOSE; } - - if (!(update->flags & REF_ISPRUNING)) - string_list_append(&refs_to_delete, - lock->ref_name); } } - if (packed_refs_lock(refs->packed_ref_store, 0, err)) { - ret = TRANSACTION_GENERIC_ERROR; - goto cleanup; - } - - if (repack_without_refs(refs->packed_ref_store, &refs_to_delete, err)) { - ret = TRANSACTION_GENERIC_ERROR; - packed_refs_unlock(refs->packed_ref_store); - goto cleanup; - } - - packed_refs_unlock(refs->packed_ref_store); - /* Delete the reflogs of any references that were deleted: */ - for_each_string_list_item(ref_to_delete, &refs_to_delete) { - strbuf_reset(&sb); - files_reflog_path(refs, &sb, ref_to_delete->string); - if (!unlink_or_warn(sb.buf)) - try_remove_empty_parents(refs, ref_to_delete->string, - REMOVE_EMPTY_PARENTS_REFLOG); + for (i = 0; i < transaction->nr; i++) { + struct ref_update *update = transaction->updates[i]; + if (update->flags & REF_DELETING && + !(update->flags & REF_LOG_ONLY) && + !(update->flags & REF_ISPRUNING)) { + strbuf_reset(&sb); + files_reflog_path(refs, &sb, update->refname); + if (!unlink_or_warn(sb.buf)) + try_remove_empty_parents(refs, update->refname, + REMOVE_EMPTY_PARENTS_REFLOG); + } } clear_loose_ref_cache(refs); cleanup: - files_transaction_cleanup(transaction); + files_transaction_cleanup(refs, transaction); for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; @@ -2639,7 +2707,6 @@ cleanup: } strbuf_release(&sb); - string_list_clear(&refs_to_delete, 0); return ret; } @@ -2647,7 +2714,10 @@ static int files_transaction_abort(struct ref_store *ref_store, struct ref_transaction *transaction, struct strbuf *err) { - files_transaction_cleanup(transaction); + struct files_ref_store *refs = + files_downcast(ref_store, 0, "ref_transaction_abort"); + + files_transaction_cleanup(refs, transaction); return 0; } diff --git a/t/t1404-update-ref-errors.sh b/t/t1404-update-ref-errors.sh index 64a8134..100d50e 100755 --- a/t/t1404-update-ref-errors.sh +++ b/t/t1404-update-ref-errors.sh @@ -404,7 +404,7 @@ test_expect_success 'broken reference blocks indirect create' ' test_cmp expected output.err ' -test_expect_failure 'no bogus intermediate values during delete' ' +test_expect_success 'no bogus intermediate values during delete' ' prefix=refs/slow-transaction && # Set up a reference with differing loose and packed versions: git update-ref $prefix/foo $C && @@ -461,7 +461,7 @@ test_expect_failure 'no bogus intermediate values during delete' ' test_must_fail git rev-parse --verify --quiet $prefix/foo ' -test_expect_failure 'delete fails cleanly if packed-refs file is locked' ' +test_expect_success 'delete fails cleanly if packed-refs file is locked' ' prefix=refs/locked-packed-refs && # Set up a reference with differing loose and packed versions: git update-ref $prefix/foo $C && -- cgit v0.10.2-6-g49f6 From 9939b33d6a3900e76b9cf95cbbb30ad8cf38cab3 Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:52 +0200 Subject: packed-backend: rip out some now-unused code Now the outside world interacts with the packed ref store only via the generic refs API plus a few lock-related functions. This allows us to delete some functions that are no longer used, thereby completing the encapsulation of the packed ref store. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 9d5f76b..0279aec 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -92,19 +92,6 @@ struct ref_store *packed_ref_store_create(const char *path, } /* - * Die if refs is not the main ref store. caller is used in any - * necessary error messages. - */ -static void packed_assert_main_repository(struct packed_ref_store *refs, - const char *caller) -{ - if (refs->store_flags & REF_STORE_MAIN) - return; - - die("BUG: operation %s only allowed for main ref store", caller); -} - -/* * Downcast `ref_store` to `packed_ref_store`. Die if `ref_store` is * not a `packed_ref_store`. Also die if `packed_ref_store` doesn't * support at least the flags specified in `required_flags`. `caller` @@ -322,40 +309,6 @@ static struct ref_dir *get_packed_refs(struct packed_ref_store *refs) } /* - * Add or overwrite a reference in the in-memory packed reference - * cache. This may only be called while the packed-refs file is locked - * (see packed_refs_lock()). To actually write the packed-refs file, - * call commit_packed_refs(). - */ -void add_packed_ref(struct ref_store *ref_store, - const char *refname, const struct object_id *oid) -{ - struct packed_ref_store *refs = - packed_downcast(ref_store, REF_STORE_WRITE, - "add_packed_ref"); - struct ref_dir *packed_refs; - struct ref_entry *packed_entry; - - if (!is_lock_file_locked(&refs->lock)) - die("BUG: packed refs not locked"); - - if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) - die("Reference has invalid format: '%s'", refname); - - packed_refs = get_packed_refs(refs); - packed_entry = find_ref_entry(packed_refs, refname); - if (packed_entry) { - /* Overwrite the existing entry: */ - oidcpy(&packed_entry->u.value.oid, oid); - packed_entry->flag = REF_ISPACKED; - oidclr(&packed_entry->u.value.peeled); - } else { - packed_entry = create_ref_entry(refname, oid, REF_ISPACKED); - add_ref_entry(packed_refs, packed_entry); - } -} - -/* * Return the ref_entry for the given refname from the packed * references. If it does not exist, return NULL. */ @@ -596,152 +549,6 @@ int packed_refs_is_locked(struct ref_store *ref_store) static const char PACKED_REFS_HEADER[] = "# pack-refs with: peeled fully-peeled \n"; -/* - * Write the current version of the packed refs cache from memory to - * disk. The packed-refs file must already be locked for writing (see - * packed_refs_lock()). Return zero on success. On errors, rollback - * the lockfile, write an error message to `err`, and return a nonzero - * value. - */ -int commit_packed_refs(struct ref_store *ref_store, struct strbuf *err) -{ - struct packed_ref_store *refs = - packed_downcast(ref_store, REF_STORE_WRITE | REF_STORE_MAIN, - "commit_packed_refs"); - struct packed_ref_cache *packed_ref_cache = - get_packed_ref_cache(refs); - int ok; - int ret = -1; - struct strbuf sb = STRBUF_INIT; - FILE *out; - struct ref_iterator *iter; - char *packed_refs_path; - - if (!is_lock_file_locked(&refs->lock)) - die("BUG: commit_packed_refs() called when unlocked"); - - /* - * If packed-refs is a symlink, we want to overwrite the - * symlinked-to file, not the symlink itself. Also, put the - * staging file next to it: - */ - packed_refs_path = get_locked_file_path(&refs->lock); - strbuf_addf(&sb, "%s.new", packed_refs_path); - if (create_tempfile(&refs->tempfile, sb.buf) < 0) { - strbuf_addf(err, "unable to create file %s: %s", - sb.buf, strerror(errno)); - strbuf_release(&sb); - goto out; - } - strbuf_release(&sb); - - out = fdopen_tempfile(&refs->tempfile, "w"); - if (!out) { - strbuf_addf(err, "unable to fdopen packed-refs tempfile: %s", - strerror(errno)); - goto error; - } - - if (fprintf(out, "%s", PACKED_REFS_HEADER) < 0) { - strbuf_addf(err, "error writing to %s: %s", - get_tempfile_path(&refs->tempfile), strerror(errno)); - goto error; - } - - iter = cache_ref_iterator_begin(packed_ref_cache->cache, NULL, 0); - while ((ok = ref_iterator_advance(iter)) == ITER_OK) { - struct object_id peeled; - int peel_error = ref_iterator_peel(iter, &peeled); - - if (write_packed_entry(out, iter->refname, iter->oid->hash, - peel_error ? NULL : peeled.hash)) { - strbuf_addf(err, "error writing to %s: %s", - get_tempfile_path(&refs->tempfile), - strerror(errno)); - ref_iterator_abort(iter); - goto error; - } - } - - if (ok != ITER_DONE) { - strbuf_addf(err, "unable to rewrite packed-refs file: " - "error iterating over old contents"); - goto error; - } - - if (rename_tempfile(&refs->tempfile, packed_refs_path)) { - strbuf_addf(err, "error replacing %s: %s", - refs->path, strerror(errno)); - goto out; - } - - ret = 0; - goto out; - -error: - delete_tempfile(&refs->tempfile); - -out: - free(packed_refs_path); - return ret; -} - -/* - * Rewrite the packed-refs file, omitting any refs listed in - * 'refnames'. On error, leave packed-refs unchanged, write an error - * message to 'err', and return a nonzero value. The packed refs lock - * must be held when calling this function; it will still be held when - * the function returns. - * - * The refs in 'refnames' needn't be sorted. `err` must not be NULL. - */ -int repack_without_refs(struct ref_store *ref_store, - struct string_list *refnames, struct strbuf *err) -{ - struct packed_ref_store *refs = - packed_downcast(ref_store, REF_STORE_WRITE | REF_STORE_MAIN, - "repack_without_refs"); - struct ref_dir *packed; - struct string_list_item *refname; - int needs_repacking = 0, removed = 0; - - packed_assert_main_repository(refs, "repack_without_refs"); - assert(err); - - if (!is_lock_file_locked(&refs->lock)) - die("BUG: repack_without_refs called without holding lock"); - - /* Look for a packed ref */ - for_each_string_list_item(refname, refnames) { - if (get_packed_ref(refs, refname->string)) { - needs_repacking = 1; - break; - } - } - - /* Avoid locking if we have nothing to do */ - if (!needs_repacking) - return 0; /* no refname exists in packed refs */ - - packed = get_packed_refs(refs); - - /* Remove refnames from the cache */ - for_each_string_list_item(refname, refnames) - if (remove_entry_from_dir(packed, refname->string) != -1) - removed = 1; - if (!removed) { - /* - * All packed entries disappeared while we were - * acquiring the lock. - */ - clear_packed_ref_cache(refs); - return 0; - } - - /* Write what remains */ - return commit_packed_refs(&refs->base, err); -} - static int packed_init_db(struct ref_store *ref_store, struct strbuf *err) { /* Nothing to do. */ diff --git a/refs/packed-backend.h b/refs/packed-backend.h index 7af2897..61687e4 100644 --- a/refs/packed-backend.h +++ b/refs/packed-backend.h @@ -23,12 +23,4 @@ int packed_refs_lock(struct ref_store *ref_store, int flags, struct strbuf *err) void packed_refs_unlock(struct ref_store *ref_store); int packed_refs_is_locked(struct ref_store *ref_store); -void add_packed_ref(struct ref_store *ref_store, - const char *refname, const struct object_id *oid); - -int commit_packed_refs(struct ref_store *ref_store, struct strbuf *err); - -int repack_without_refs(struct ref_store *ref_store, - struct string_list *refnames, struct strbuf *err); - #endif /* REFS_PACKED_BACKEND_H */ -- cgit v0.10.2-6-g49f6 From 5e00a6c873981f87165adfecf29ad0ecc2c2c5df Mon Sep 17 00:00:00 2001 From: Michael Haggerty Date: Fri, 8 Sep 2017 15:51:53 +0200 Subject: files_transaction_finish(): delete reflogs before references If the deletion steps unexpectedly fail, it is less bad to leave a reference without its reflog than it is to leave a reflog without its reference, since the latter is an invalid repository state. Signed-off-by: Michael Haggerty Signed-off-by: Junio C Hamano diff --git a/refs/files-backend.c b/refs/files-backend.c index 29eb5e8..961424a 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2637,6 +2637,27 @@ static int files_transaction_finish(struct ref_store *ref_store, } /* + * Now that updates are safely completed, we can perform + * deletes. First delete the reflogs of any references that + * will be deleted, since (in the unexpected event of an + * error) leaving a reference without a reflog is less bad + * than leaving a reflog without a reference (the latter is a + * mildly invalid repository state): + */ + for (i = 0; i < transaction->nr; i++) { + struct ref_update *update = transaction->updates[i]; + if (update->flags & REF_DELETING && + !(update->flags & REF_LOG_ONLY) && + !(update->flags & REF_ISPRUNING)) { + strbuf_reset(&sb); + files_reflog_path(refs, &sb, update->refname); + if (!unlink_or_warn(sb.buf)) + try_remove_empty_parents(refs, update->refname, + REMOVE_EMPTY_PARENTS_REFLOG); + } + } + + /* * Perform deletes now that updates are safely completed. * * First delete any packed versions of the references, while @@ -2672,20 +2693,6 @@ static int files_transaction_finish(struct ref_store *ref_store, } } - /* Delete the reflogs of any references that were deleted: */ - for (i = 0; i < transaction->nr; i++) { - struct ref_update *update = transaction->updates[i]; - if (update->flags & REF_DELETING && - !(update->flags & REF_LOG_ONLY) && - !(update->flags & REF_ISPRUNING)) { - strbuf_reset(&sb); - files_reflog_path(refs, &sb, update->refname); - if (!unlink_or_warn(sb.buf)) - try_remove_empty_parents(refs, update->refname, - REMOVE_EMPTY_PARENTS_REFLOG); - } - } - clear_loose_ref_cache(refs); cleanup: -- cgit v0.10.2-6-g49f6