From 522e6417487cc5c3f2f6d49c8f63554af63d8eda Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:31 +0000 Subject: t1091: use check_files to reduce boilerplate When testing the sparse-checkout feature, we need to compare the contents of the working-directory against some expected output. Using here-docs was useful in the beginning, but became repetetive as the test script grew. Create a check_files helper to make the tests simpler and easier to extend. It also reduces instances of bad here-doc whitespace. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index ff7f8f7..e058a20 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -12,6 +12,13 @@ list_files() { (cd "$1" && printf '%s\n' *) } +check_files() { + list_files "$1" >actual && + shift && + printf "%s\n" $@ >expect && + test_cmp expect actual +} + test_expect_success 'setup' ' git init repo && ( @@ -58,9 +65,7 @@ test_expect_success 'git sparse-checkout init' ' EOF test_cmp expect repo/.git/info/sparse-checkout && test_cmp_config -C repo true core.sparsecheckout && - list_files repo >dir && - echo a >expect && - test_cmp expect dir + check_files repo a ' test_expect_success 'git sparse-checkout list after init' ' @@ -81,13 +86,7 @@ test_expect_success 'init with existing sparse-checkout' ' *folder* EOF test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'clone --sparse' ' @@ -98,9 +97,7 @@ test_expect_success 'clone --sparse' ' !/*/ EOF test_cmp expect actual && - list_files clone >dir && - echo a >expect && - test_cmp expect dir + check_files clone a ' test_expect_success 'set enables config' ' @@ -127,13 +124,7 @@ test_expect_success 'set sparse-checkout using builtin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'set sparse-checkout using --stdin' ' @@ -147,13 +138,7 @@ test_expect_success 'set sparse-checkout using --stdin' ' git -C repo sparse-checkout list >actual && test_cmp expect actual && test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo "a folder1 folder2" ' test_expect_success 'cone mode: match patterns' ' @@ -162,13 +147,7 @@ test_expect_success 'cone mode: match patterns' ' git -C repo read-tree -mu HEAD 2>err && test_i18ngrep ! "disabling cone patterns" err && git -C repo reset --hard && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'cone mode: warn on bad pattern' ' @@ -185,14 +164,7 @@ test_expect_success 'sparse-checkout disable' ' test_path_is_file repo/.git/info/sparse-checkout && git -C repo config --list >config && test_must_fail git config core.sparseCheckout && - list_files repo >dir && - cat >expect <<-EOF && - a - deep - folder1 - folder2 - EOF - test_cmp expect dir + check_files repo a deep folder1 folder2 ' test_expect_success 'cone mode: init and set' ' @@ -204,24 +176,9 @@ test_expect_success 'cone mode: init and set' ' test_cmp expect dir && git -C repo sparse-checkout set deep/deeper1/deepest/ 2>err && test_must_be_empty err && - list_files repo >dir && - cat >expect <<-EOF && - a - deep - EOF - test_cmp expect dir && - list_files repo/deep >dir && - cat >expect <<-EOF && - a - deeper1 - EOF - test_cmp expect dir && - list_files repo/deep/deeper1 >dir && - cat >expect <<-EOF && - a - deepest - EOF - test_cmp expect dir && + check_files repo a deep && + check_files repo/deep a deeper1 && + check_files repo/deep/deeper1 a deepest && cat >expect <<-EOF && /* !/*/ @@ -237,13 +194,7 @@ test_expect_success 'cone mode: init and set' ' folder2 EOF test_must_be_empty err && - cat >expect <<-EOF && - a - folder1 - folder2 - EOF - list_files repo >dir && - test_cmp expect dir + check_files repo a folder1 folder2 ' test_expect_success 'cone mode: list' ' @@ -275,13 +226,7 @@ test_expect_success 'revert to old sparse-checkout on bad update' ' test_must_fail git -C repo sparse-checkout set deep/deeper1 2>err && test_i18ngrep "cannot set sparse-checkout patterns" err && test_cmp repo/.git/info/sparse-checkout expect && - list_files repo/deep >dir && - cat >expect <<-EOF && - a - deeper1 - deeper2 - EOF - test_cmp dir expect + check_files repo/deep a deeper1 deeper2 ' test_expect_success 'revert to old sparse-checkout on empty update' ' @@ -332,12 +277,7 @@ test_expect_success 'cone mode: set with core.ignoreCase=true' ' /folder1/ EOF test_cmp expect repo/.git/info/sparse-checkout && - list_files repo >dir && - cat >expect <<-EOF && - a - folder1 - EOF - test_cmp expect dir + check_files repo a folder1 ' test_expect_success 'interaction with submodules' ' @@ -351,21 +291,8 @@ test_expect_success 'interaction with submodules' ' git sparse-checkout init --cone && git sparse-checkout set folder1 ) && - list_files super >dir && - cat >expect <<-\EOF && - a - folder1 - modules - EOF - test_cmp expect dir && - list_files super/modules/child >dir && - cat >expect <<-\EOF && - a - deep - folder1 - folder2 - EOF - test_cmp expect dir + check_files super a folder1 modules && + check_files super/modules/child a deep folder1 folder2 ' test_done -- cgit v0.10.2-6-g49f6 From d622c34396b3ea1a81f07d951ee1112f83d9330c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:32 +0000 Subject: t1091: improve here-docs t1091-sparse-checkout-builtin.sh uses here-docs to populate the expected contents of the sparse-checkout file. These do not use shell interpolation, so use "-\EOF" instead of "-EOF". Also use proper tabbing. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e058a20..e28e1c7 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -46,11 +46,11 @@ test_expect_success 'git sparse-checkout list (empty)' ' test_expect_success 'git sparse-checkout list (populated)' ' test_when_finished rm -f repo/.git/info/sparse-checkout && - cat >repo/.git/info/sparse-checkout <<-EOF && - /folder1/* - /deep/ - **/a - !*bin* + cat >repo/.git/info/sparse-checkout <<-\EOF && + /folder1/* + /deep/ + **/a + !*bin* EOF cp repo/.git/info/sparse-checkout expect && git -C repo sparse-checkout list >list && @@ -59,9 +59,9 @@ test_expect_success 'git sparse-checkout list (populated)' ' test_expect_success 'git sparse-checkout init' ' git -C repo sparse-checkout init && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect repo/.git/info/sparse-checkout && test_cmp_config -C repo true core.sparsecheckout && @@ -70,9 +70,9 @@ test_expect_success 'git sparse-checkout init' ' test_expect_success 'git sparse-checkout list after init' ' git -C repo sparse-checkout list >actual && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect actual ' @@ -80,10 +80,10 @@ test_expect_success 'git sparse-checkout list after init' ' test_expect_success 'init with existing sparse-checkout' ' echo "*folder*" >> repo/.git/info/sparse-checkout && git -C repo sparse-checkout init && - cat >expect <<-EOF && - /* - !/*/ - *folder* + cat >expect <<-\EOF && + /* + !/*/ + *folder* EOF test_cmp expect repo/.git/info/sparse-checkout && check_files repo a folder1 folder2 @@ -92,9 +92,9 @@ test_expect_success 'init with existing sparse-checkout' ' test_expect_success 'clone --sparse' ' git clone --sparse repo clone && git -C clone sparse-checkout list >actual && - cat >expect <<-EOF && - /* - !/*/ + cat >expect <<-\EOF && + /* + !/*/ EOF test_cmp expect actual && check_files clone a @@ -116,10 +116,10 @@ test_expect_success 'set enables config' ' test_expect_success 'set sparse-checkout using builtin' ' git -C repo sparse-checkout set "/*" "!/*/" "*folder*" && - cat >expect <<-EOF && - /* - !/*/ - *folder* + cat >expect <<-\EOF && + /* + !/*/ + *folder* EOF git -C repo sparse-checkout list >actual && test_cmp expect actual && @@ -128,11 +128,11 @@ test_expect_success 'set sparse-checkout using builtin' ' ' test_expect_success 'set sparse-checkout using --stdin' ' - cat >expect <<-EOF && - /* - !/*/ - /folder1/ - /folder2/ + cat >expect <<-\EOF && + /* + !/*/ + /folder1/ + /folder2/ EOF git -C repo sparse-checkout set --stdin actual && @@ -179,28 +179,28 @@ test_expect_success 'cone mode: init and set' ' check_files repo a deep && check_files repo/deep a deeper1 && check_files repo/deep/deeper1 a deepest && - cat >expect <<-EOF && - /* - !/*/ - /deep/ - !/deep/*/ - /deep/deeper1/ - !/deep/deeper1/*/ - /deep/deeper1/deepest/ + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + !/deep/*/ + /deep/deeper1/ + !/deep/deeper1/*/ + /deep/deeper1/deepest/ EOF test_cmp expect repo/.git/info/sparse-checkout && - git -C repo sparse-checkout set --stdin 2>err <<-EOF && - folder1 - folder2 + git -C repo sparse-checkout set --stdin 2>err <<-\EOF && + folder1 + folder2 EOF test_must_be_empty err && check_files repo a folder1 folder2 ' test_expect_success 'cone mode: list' ' - cat >expect <<-EOF && - folder1 - folder2 + cat >expect <<-\EOF && + folder1 + folder2 EOF git -C repo sparse-checkout set --stdin actual 2>err && @@ -211,10 +211,10 @@ test_expect_success 'cone mode: list' ' test_expect_success 'cone mode: set with nested folders' ' git -C repo sparse-checkout set deep deep/deeper1/deepest 2>err && test_line_count = 0 err && - cat >expect <<-EOF && - /* - !/*/ - /deep/ + cat >expect <<-\EOF && + /* + !/*/ + /deep/ EOF test_cmp repo/.git/info/sparse-checkout expect ' @@ -271,10 +271,10 @@ test_expect_success 'sparse-checkout (init|set|disable) fails with dirty status' test_expect_success 'cone mode: set with core.ignoreCase=true' ' git -C repo sparse-checkout init --cone && git -C repo -c core.ignoreCase=true sparse-checkout set folder1 && - cat >expect <<-EOF && - /* - !/*/ - /folder1/ + cat >expect <<-\EOF && + /* + !/*/ + /folder1/ EOF test_cmp expect repo/.git/info/sparse-checkout && check_files repo a folder1 -- cgit v0.10.2-6-g49f6 From 3c754067a1164ffafd965dcd44a9f004e6100e42 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:33 +0000 Subject: sparse-checkout: create leading directories The 'git init' command creates the ".git/info" directory and fills it with some default files. However, 'git worktree add' does not create the info directory for that worktree. This causes a problem when running "git sparse-checkout init" inside a worktree. While care was taken to allow the sparse-checkout config to be specific to a worktree, this initialization was untested. Safely create the leading directories for the sparse-checkout file. This is the safest thing to do even without worktrees, as a user could delete their ".git/info" directory and expect Git to recover safely. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index b3bed89..3cee8ab 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -199,6 +199,10 @@ static int write_patterns_and_update(struct pattern_list *pl) int result; sparse_filename = get_sparse_checkout_filename(); + + if (safe_create_leading_directories(sparse_filename)) + die(_("failed to create directory for sparse-checkout file")); + fd = hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e28e1c7..43d1f75 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -295,4 +295,14 @@ test_expect_success 'interaction with submodules' ' check_files super/modules/child a deep folder1 folder2 ' +test_expect_success 'different sparse-checkouts with worktrees' ' + git -C repo worktree add --detach ../worktree && + check_files worktree "a deep folder1 folder2" && + git -C worktree sparse-checkout init --cone && + git -C repo sparse-checkout set folder1 && + git -C worktree sparse-checkout set deep/deeper1 && + check_files repo a folder1 && + check_files worktree a deep +' + test_done -- cgit v0.10.2-6-g49f6 From 47dbf10d8a5ce8c9b441a16b7698c7d70585dff0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:34 +0000 Subject: clone: fix --sparse option with URLs The --sparse option was added to the clone builtin in d89f09c (clone: add --sparse mode, 2019-11-21) and was tested with a local path clone in t1091-sparse-checkout-builtin.sh. However, due to a difference in how local paths are handled versus URLs, this mechanism does not work with URLs. Modify the test to use a "file://" URL, which would output this error before the code change: Cloning into 'clone'... fatal: cannot change to 'file://.../repo': No such file or directory error: failed to initialize sparse-checkout These errors are due to using a "-C " option to call 'git -C sparse-checkout init' but the URL is being given instead of the target directory. Update that target directory to evaluate this correctly. I have also manually tested that https:// URLs are handled correctly as well. Acked-by: Taylor Blau Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/clone.c b/builtin/clone.c index 4348d96..2caefc4 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1130,7 +1130,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (option_required_reference.nr || option_optional_reference.nr) setup_reference(); - if (option_sparse_checkout && git_sparse_checkout_init(repo)) + if (option_sparse_checkout && git_sparse_checkout_init(dir)) return 1; remote = remote_get(option_origin); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 43d1f75..cf4a595 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -90,7 +90,7 @@ test_expect_success 'init with existing sparse-checkout' ' ' test_expect_success 'clone --sparse' ' - git clone --sparse repo clone && + git clone --sparse "file://$(pwd)/repo" clone && git -C clone sparse-checkout list >actual && cat >expect <<-\EOF && /* -- cgit v0.10.2-6-g49f6 From 7aa9ef2fcaa986d7f11064adab6d1c010d4f2ead Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 24 Jan 2020 21:19:35 +0000 Subject: sparse-checkout: fix documentation typo for core.sparseCheckoutCone Signed-off-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 3b341cf..4834fb4 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -106,7 +106,7 @@ The full pattern set allows for arbitrary pattern matches and complicated inclusion/exclusion rules. These can result in O(N*M) pattern matches when updating the index, where N is the number of patterns and M is the number of paths in the index. To combat this performance issue, a more restricted -pattern set is allowed when `core.spareCheckoutCone` is enabled. +pattern set is allowed when `core.sparseCheckoutCone` is enabled. The accepted patterns in the cone pattern set are: -- cgit v0.10.2-6-g49f6 From 41de0c6fbcc3d2544ebada3a9f26dec0f32f42de Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:36 +0000 Subject: sparse-checkout: cone mode does not recognize "**" When core.sparseCheckoutCone is enabled, the 'git sparse-checkout set' command creates a restricted set of possible patterns that are used by a custom algorithm to quickly match those patterns. If a user manually edits the sparse-checkout file, then they could create patterns that do not match these expectations. The cone-mode matching algorithm can return incorrect results. The solution is to detect these incorrect patterns, warn that we do not recognize them, and revert to the standard algorithm. Check each pattern for the "**" substring, and revert to the old logic if seen. While technically a "//**" pattern matches the meaning of "//", it is not one that would be written by the sparse-checkout builtin in cone mode. Attempting to accept that pattern change complicates the logic and instead we punt and do not accept any instance of "**". Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/dir.c b/dir.c index 22d08e6..40fed73 100644 --- a/dir.c +++ b/dir.c @@ -651,11 +651,16 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern return; } + if (strstr(given->pattern, "**")) { + /* Not a cone pattern. */ + warning(_("unrecognized pattern: '%s'"), given->pattern); + goto clear_hashmaps; + } + if (given->patternlen > 2 && !strcmp(given->pattern + given->patternlen - 2, "/*")) { if (!(given->flags & PATTERN_FLAG_NEGATIVE)) { /* Not a cone pattern. */ - pl->use_cone_patterns = 0; warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index cf4a595..e2e45dc 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -305,4 +305,38 @@ test_expect_success 'different sparse-checkouts with worktrees' ' check_files worktree a deep ' +check_read_tree_errors () { + REPO=$1 + FILES=$2 + ERRORS=$3 + git -C $REPO read-tree -mu HEAD 2>err && + if test -z "$ERRORS" + then + test_must_be_empty err + else + test_i18ngrep "$ERRORS" err + fi && + check_files $REPO $FILES +} + +test_expect_success 'pattern-checks: /A/**' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /folder1/** + EOF + check_read_tree_errors repo "a folder1" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: /A/**/B/' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /deep/**/deepest + EOF + check_read_tree_errors repo "a deep" "disabling cone pattern matching" && + check_files repo/deep "deeper1" && + check_files repo/deep/deeper1 "deepest" +' + test_done -- cgit v0.10.2-6-g49f6 From 9e6d3e64175713bc0007f3012ea288f4dfc0a399 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 24 Jan 2020 21:19:37 +0000 Subject: sparse-checkout: detect short patterns In cone mode, the shortest pattern the sparse-checkout command will write into the sparse-checkout file is "/*". This is handled carefully in add_pattern_to_hashsets(), so warn if any other pattern is this short. This will assist future pattern checks by allowing us to assume there are at least three characters in the pattern. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/dir.c b/dir.c index 40fed73..c2e5856 100644 --- a/dir.c +++ b/dir.c @@ -651,7 +651,8 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern return; } - if (strstr(given->pattern, "**")) { + if (given->patternlen <= 2 || + strstr(given->pattern, "**")) { /* Not a cone pattern. */ warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index e2e45dc..2e57534 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -339,4 +339,13 @@ test_expect_success 'pattern-checks: /A/**/B/' ' check_files repo/deep/deeper1 "deepest" ' +test_expect_success 'pattern-checks: too short' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /a + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" +' + test_done -- cgit v0.10.2-6-g49f6 From 9abc60f8015d060d3f3433b105648a4725c97bd1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:08 +0000 Subject: sparse-checkout: warn on globs in cone patterns In cone mode, the sparse-checkout commmand will write patterns that allow faster pattern matching. This matching only works if the patterns in the sparse-checkout file are those written by that command. Users can edit the sparse-checkout file and create patterns that cause the cone mode matching to fail. The cone mode patterns may end in "/*" but otherwise an un-escaped asterisk or other glob character is invalid. Add checks to disable cone mode when seeing these values. A later change will properly handle escaped globs. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/dir.c b/dir.c index c2e5856..71d2833 100644 --- a/dir.c +++ b/dir.c @@ -635,6 +635,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern struct pattern_entry *translated; char *truncated; char *data = NULL; + const char *prev, *cur, *next; if (!pl->use_cone_patterns) return; @@ -652,12 +653,47 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern } if (given->patternlen <= 2 || + *given->pattern == '*' || strstr(given->pattern, "**")) { /* Not a cone pattern. */ warning(_("unrecognized pattern: '%s'"), given->pattern); goto clear_hashmaps; } + prev = given->pattern; + cur = given->pattern + 1; + next = given->pattern + 2; + + while (*cur) { + /* Watch for glob characters '*', '\', '[', '?' */ + if (!is_glob_special(*cur)) + goto increment; + + /* But only if *prev != '\\' */ + if (*prev == '\\') + goto increment; + + /* But allow the initial '\' */ + if (*cur == '\\' && + is_glob_special(*next)) + goto increment; + + /* But a trailing '/' then '*' is fine */ + if (*prev == '/' && + *cur == '*' && + *next == 0) + goto increment; + + /* Not a cone pattern. */ + warning(_("unrecognized pattern: '%s'"), given->pattern); + goto clear_hashmaps; + + increment: + prev++; + cur++; + next++; + } + if (given->patternlen > 2 && !strcmp(given->pattern + given->patternlen - 2, "/*")) { if (!(given->flags & PATTERN_FLAG_NEGATIVE)) { diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 2e57534..c732abe 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -348,4 +348,43 @@ test_expect_success 'pattern-checks: too short' ' check_read_tree_errors repo "a" "disabling cone pattern matching" ' +test_expect_success 'pattern-checks: trailing "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /a* + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: starting "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + *eep/ + EOF + check_read_tree_errors repo "a deep" "disabling cone pattern matching" +' + +test_expect_success 'pattern-checks: contained glob characters' ' + for c in "[a]" "\\" "?" "*" + do + cat >repo/.git/info/sparse-checkout <<-EOF && + /* + !/*/ + something$c-else/ + EOF + check_read_tree_errors repo "a" "disabling cone pattern matching" + done +' + +test_expect_success 'pattern-checks: escaped "*"' ' + cat >repo/.git/info/sparse-checkout <<-\EOF && + /* + !/*/ + /does\*not\*exist/ + EOF + check_read_tree_errors repo "a" "" +' + test_done -- cgit v0.10.2-6-g49f6 From 4f52c2ce6c578896964e960f6017510f0efd3f46 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:09 +0000 Subject: sparse-checkout: properly match escaped characters In cone mode, the sparse-checkout feature uses hashset containment queries to match paths. Make this algorithm respect escaped asterisk (*) and backslash (\) characters. Create dup_and_filter_pattern() method to convert a pattern by removing escape characters and dropping an optional "/*" at the end. This method is available in dir.h as we will use it in builtin/sparse-checkout.c in a later change. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/dir.c b/dir.c index 71d2833..7ac0920 100644 --- a/dir.c +++ b/dir.c @@ -630,6 +630,36 @@ int pl_hashmap_cmp(const void *unused_cmp_data, return strncmp(ee1->pattern, ee2->pattern, min_len); } +static char *dup_and_filter_pattern(const char *pattern) +{ + char *set, *read; + size_t count = 0; + char *result = xstrdup(pattern); + + set = result; + read = result; + + while (*read) { + /* skip escape characters (once) */ + if (*read == '\\') + read++; + + *set = *read; + + set++; + read++; + count++; + } + *set = 0; + + if (count > 2 && + *(set - 1) == '*' && + *(set - 2) == '/') + *(set - 2) = 0; + + return result; +} + static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern *given) { struct pattern_entry *translated; @@ -702,8 +732,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern goto clear_hashmaps; } - truncated = xstrdup(given->pattern); - truncated[given->patternlen - 2] = 0; + truncated = dup_and_filter_pattern(given->pattern); translated = xmalloc(sizeof(struct pattern_entry)); translated->pattern = truncated; @@ -737,7 +766,7 @@ static void add_pattern_to_hashsets(struct pattern_list *pl, struct path_pattern translated = xmalloc(sizeof(struct pattern_entry)); - translated->pattern = xstrdup(given->pattern); + translated->pattern = dup_and_filter_pattern(given->pattern); translated->patternlen = given->patternlen; hashmap_entry_init(&translated->ent, ignore_case ? diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index c732abe..9ea7008 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -378,13 +378,28 @@ test_expect_success 'pattern-checks: contained glob characters' ' done ' -test_expect_success 'pattern-checks: escaped "*"' ' - cat >repo/.git/info/sparse-checkout <<-\EOF && +test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' + git clone repo escaped && + TREEOID=$(git -C escaped rev-parse HEAD:folder1) && + NEWTREE=$(git -C escaped mktree <<-EOF + $(git -C escaped ls-tree HEAD) + 040000 tree $TREEOID zbad\\dir + 040000 tree $TREEOID zdoes*exist + EOF + ) && + COMMIT=$(git -C escaped commit-tree $NEWTREE -p HEAD) && + git -C escaped reset --hard $COMMIT && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + git -C escaped sparse-checkout init --cone && + cat >escaped/.git/info/sparse-checkout <<-\EOF && /* !/*/ - /does\*not\*exist/ + /zbad\\dir/ + !/zbad\\dir/*/ + /zdoes\*not\*exist/ + /zdoes\*exist/ EOF - check_read_tree_errors repo "a" "" + check_read_tree_errors escaped "a zbad\\dir zdoes*exist" ' test_done -- cgit v0.10.2-6-g49f6 From d585f0e7992ea7f025a5a91f46f2baa9e88f19f6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:10 +0000 Subject: sparse-checkout: write escaped patterns in cone mode If a user somehow creates a directory with an asterisk (*) or backslash (\), then the "git sparse-checkout set" command will struggle to provide the correct pattern in the sparse-checkout file. When not in cone mode, the provided pattern is written directly into the sparse-checkout file. However, in cone mode we expect a list of paths to directories and then we convert those into patterns. However, there is some care needed for the timing of these escapes. The in-memory pattern list is used to update the working directory before writing the patterns to disk. Thus, we need the command to have the unescaped names in the hashsets for the cone comparisons, then escape the patterns later. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 3cee8ab..cc86b8a 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -13,6 +13,7 @@ #include "resolve-undo.h" #include "unpack-trees.h" #include "wt-status.h" +#include "quote.h" static const char *empty_base = ""; @@ -140,6 +141,22 @@ static int update_working_directory(struct pattern_list *pl) return result; } +static char *escaped_pattern(char *pattern) +{ + char *p = pattern; + struct strbuf final = STRBUF_INIT; + + while (*p) { + if (*p == '*' || *p == '\\') + strbuf_addch(&final, '\\'); + + strbuf_addch(&final, *p); + p++; + } + + return strbuf_detach(&final, NULL); +} + static void write_cone_to_file(FILE *fp, struct pattern_list *pl) { int i; @@ -164,10 +181,11 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) fprintf(fp, "/*\n!/*/\n"); for (i = 0; i < sl.nr; i++) { - char *pattern = sl.items[i].string; + char *pattern = escaped_pattern(sl.items[i].string); if (strlen(pattern)) fprintf(fp, "%s/\n!%s/*/\n", pattern, pattern); + free(pattern); } string_list_clear(&sl, 0); @@ -185,8 +203,9 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) string_list_remove_duplicates(&sl, 0); for (i = 0; i < sl.nr; i++) { - char *pattern = sl.items[i].string; + char *pattern = escaped_pattern(sl.items[i].string); fprintf(fp, "%s/\n", pattern); + free(pattern); } } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 9ea7008..fb8718e 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -309,6 +309,9 @@ check_read_tree_errors () { REPO=$1 FILES=$2 ERRORS=$3 + git -C $REPO -c core.sparseCheckoutCone=false read-tree -mu HEAD 2>err && + test_must_be_empty err && + check_files $REPO "$FILES" && git -C $REPO read-tree -mu HEAD 2>err && if test -z "$ERRORS" then @@ -391,14 +394,17 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' git -C escaped reset --hard $COMMIT && check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && git -C escaped sparse-checkout init --cone && - cat >escaped/.git/info/sparse-checkout <<-\EOF && + git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" && + cat >expect <<-\EOF && /* !/*/ /zbad\\dir/ !/zbad\\dir/*/ - /zdoes\*not\*exist/ + /zbad\\dir/bogus/ /zdoes\*exist/ + /zdoes\*not\*exist/ EOF + test_cmp expect escaped/.git/info/sparse-checkout && check_read_tree_errors escaped "a zbad\\dir zdoes*exist" ' -- cgit v0.10.2-6-g49f6 From bd64de42de28e5cdda7765d5de1c3ed34d4898cb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:11 +0000 Subject: sparse-checkout: unquote C-style strings over --stdin If a user somehow creates a directory with an asterisk (*) or backslash (\), then the "git sparse-checkout set" command will struggle to provide the correct pattern in the sparse-checkout file. When not in cone mode, the provided pattern is written directly into the sparse-checkout file. However, in cone mode we expect a list of paths to directories and then we convert those into patterns. Even more specifically, the goal is to always allow the following from the root of a repo: git ls-tree --name-only -d HEAD | git sparse-checkout set --stdin The ls-tree command provides directory names with an unescaped asterisk. It also quotes the directories that contain an escaped backslash. We must remove these quotes, then keep the escaped backslashes. Use unquote_c_style() when parsing lines from stdin. Command-line arguments will be parsed as-is, assuming the user can do the correct level of escaping from their environment to match the exact directory names. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index cc86b8a..6083aa1 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -442,8 +442,21 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix) pl.use_cone_patterns = 1; if (set_opts.use_stdin) { - while (!strbuf_getline(&line, stdin)) + struct strbuf unquoted = STRBUF_INIT; + while (!strbuf_getline(&line, stdin)) { + if (line.buf[0] == '"') { + strbuf_reset(&unquoted); + if (unquote_c_style(&unquoted, line.buf, NULL)) + die(_("unable to unquote C-style string '%s'"), + line.buf); + + strbuf_swap(&unquoted, &line); + } + strbuf_to_cone_pattern(&line, &pl); + } + + strbuf_release(&unquoted); } else { for (i = 0; i < argc; i++) { strbuf_setlen(&line, 0); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index fb8718e..a46a310 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -405,7 +405,19 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zdoes\*not\*exist/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_read_tree_errors escaped "a zbad\\dir zdoes*exist" + check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && + git -C escaped ls-tree -d --name-only HEAD | git -C escaped sparse-checkout set --stdin && + cat >expect <<-\EOF && + /* + !/*/ + /deep/ + /folder1/ + /folder2/ + /zbad\\dir/ + /zdoes\*exist/ + EOF + test_cmp expect escaped/.git/info/sparse-checkout && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" ' test_done -- cgit v0.10.2-6-g49f6 From e55682ea2640dd3aa002a2657c32bdd1d85b44e9 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:12 +0000 Subject: sparse-checkout: use C-style quotes in 'list' subcommand When in cone mode, the 'git sparse-checkout list' subcommand lists the directories included in the sparse cone. When these directories contain odd characters, such as a backslash, then we need to use C-style quotes similar to 'git ls-tree'. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 6083aa1..facdb6b 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -78,8 +78,10 @@ static int sparse_checkout_list(int argc, const char **argv) string_list_sort(&sl); - for (i = 0; i < sl.nr; i++) - printf("%s\n", sl.items[i].string); + for (i = 0; i < sl.nr; i++) { + quote_c_style(sl.items[i].string, NULL, stdout, 0); + printf("\n"); + } return 0; } diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index a46a310..545e8d5 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -406,7 +406,8 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' EOF test_cmp expect escaped/.git/info/sparse-checkout && check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && - git -C escaped ls-tree -d --name-only HEAD | git -C escaped sparse-checkout set --stdin && + git -C escaped ls-tree -d --name-only HEAD >list-expect && + git -C escaped sparse-checkout set --stdin expect <<-\EOF && /* !/*/ @@ -417,7 +418,9 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zdoes\*exist/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + git -C escaped sparse-checkout list >list-actual && + test_cmp list-expect list-actual ' test_done -- cgit v0.10.2-6-g49f6 From e53ffe2704d7e10690f4382e46c1411a482531f1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:13 +0000 Subject: sparse-checkout: escape all glob characters on write The sparse-checkout patterns allow special globs according to fnmatch(3). When writing cone-mode patterns for paths containing these characters, they must be escaped. Use is_glob_special() to check which characters must be escaped this way, and add a path to the tests that contains all glob characters at once. Note that ']' is not special, since the initial bracket '[' is escaped. Reported-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index facdb6b..7aeb384 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -149,7 +149,7 @@ static char *escaped_pattern(char *pattern) struct strbuf final = STRBUF_INIT; while (*p) { - if (*p == '*' || *p == '\\') + if (is_glob_special(*p)) strbuf_addch(&final, '\\'); strbuf_addch(&final, *p); diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 545e8d5..37e9304 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -381,20 +381,21 @@ test_expect_success 'pattern-checks: contained glob characters' ' done ' -test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' +test_expect_success BSLASHPSPEC 'pattern-checks: escaped characters' ' git clone repo escaped && TREEOID=$(git -C escaped rev-parse HEAD:folder1) && NEWTREE=$(git -C escaped mktree <<-EOF $(git -C escaped ls-tree HEAD) 040000 tree $TREEOID zbad\\dir 040000 tree $TREEOID zdoes*exist + 040000 tree $TREEOID zglob[!a]? EOF ) && COMMIT=$(git -C escaped commit-tree $NEWTREE -p HEAD) && git -C escaped reset --hard $COMMIT && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? && git -C escaped sparse-checkout init --cone && - git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" && + git -C escaped sparse-checkout set zbad\\dir/bogus "zdoes*not*exist" "zdoes*exist" "zglob[!a]?" && cat >expect <<-\EOF && /* !/*/ @@ -403,9 +404,10 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /zbad\\dir/bogus/ /zdoes\*exist/ /zdoes\*not\*exist/ + /zglob\[!a]\?/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_read_tree_errors escaped "a zbad\\dir zdoes*exist" && + check_read_tree_errors escaped "a zbad\\dir zdoes*exist zglob[!a]?" && git -C escaped ls-tree -d --name-only HEAD >list-expect && git -C escaped sparse-checkout set --stdin expect <<-\EOF && @@ -416,9 +418,10 @@ test_expect_success BSLASHPSPEC 'pattern-checks: escaped "*"' ' /folder2/ /zbad\\dir/ /zdoes\*exist/ + /zglob\[!a]\?/ EOF test_cmp expect escaped/.git/info/sparse-checkout && - check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" && + check_files escaped "a deep folder1 folder2 zbad\\dir zdoes*exist" zglob[!a]? && git -C escaped sparse-checkout list >list-actual && test_cmp list-expect list-actual ' -- cgit v0.10.2-6-g49f6 From d2e65f4c9056be72ff8a1f39245c5e1b27d556b2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:14 +0000 Subject: sparse-checkout: improve docs around 'set' in cone mode The existing documentation does not clarify how the 'set' subcommand changes when core.sparseCheckoutCone is enabled. Correct this by changing some language around the "A/B/C" example. Also include a description of the input format matching the output of 'git ls-tree --name-only'. Helped-by: Jeff King Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/Documentation/git-sparse-checkout.txt b/Documentation/git-sparse-checkout.txt index 4834fb4..0914619 100644 --- a/Documentation/git-sparse-checkout.txt +++ b/Documentation/git-sparse-checkout.txt @@ -50,6 +50,14 @@ To avoid interfering with other worktrees, it first enables the + When the `--stdin` option is provided, the patterns are read from standard in as a newline-delimited list instead of from the arguments. ++ +When `core.sparseCheckoutCone` is enabled, the input list is considered a +list of directories instead of sparse-checkout patterns. The command writes +patterns to the sparse-checkout file to include all files contained in those +directories (recursively) as well as files that are siblings of ancestor +directories. The input format matches the output of `git ls-tree --name-only`. +This includes interpreting pathnames that begin with a double quote (") as +C-style quoted strings. 'disable':: Disable the `core.sparseCheckout` config setting, and restore the @@ -128,9 +136,12 @@ the following patterns: ---------------- This says "include everything in root, but nothing two levels below root." -If we then add the folder `A/B/C` as a recursive pattern, the folders `A` and -`A/B` are added as parent patterns. The resulting sparse-checkout file is -now + +When in cone mode, the `git sparse-checkout set` subcommand takes a list of +directories instead of a list of sparse-checkout patterns. In this mode, +the command `git sparse-checkout set A/B/C` sets the directory `A/B/C` as +a recursive pattern, the directories `A` and `A/B` are added as parent +patterns. The resulting sparse-checkout file is now ---------------- /* -- cgit v0.10.2-6-g49f6 From f998a3f1e588d73ed7285cb14ac4839f63f6dc82 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 31 Jan 2020 20:16:15 +0000 Subject: sparse-checkout: fix cone mode behavior mismatch The intention of the special "cone mode" in the sparse-checkout feature is to always match the same patterns that are matched by the same sparse-checkout file as when cone mode is disabled. When a file path is given to "git sparse-checkout set" in cone mode, then the cone mode improperly matches the file as a recursive path. When setting the skip-worktree bits, files were not expecting the MATCHED_RECURSIVE response, and hence these were left out of the matched cone. Fix this bug by checking for MATCHED_RECURSIVE in addition to MATCHED and add a test that prevents regression. Reported-by: Finn Bryant Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index 37e9304..7d98209 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -305,6 +305,18 @@ test_expect_success 'different sparse-checkouts with worktrees' ' check_files worktree a deep ' +test_expect_success 'set using filename keeps file on-disk' ' + git -C repo sparse-checkout set a deep && + cat >expect <<-\EOF && + /* + !/*/ + /a/ + /deep/ + EOF + test_cmp expect repo/.git/info/sparse-checkout && + check_files repo a deep +' + check_read_tree_errors () { REPO=$1 FILES=$2 diff --git a/unpack-trees.c b/unpack-trees.c index 3789a22..78425ce 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1416,7 +1416,7 @@ static int clear_ce_flags_1(struct index_state *istate, name, &dtype, pl, istate); if (ret == UNDECIDED) ret = default_match; - if (ret == MATCHED) + if (ret == MATCHED || ret == MATCHED_RECURSIVE) ce->ce_flags &= ~clear_mask; cache++; progress_nr++; -- cgit v0.10.2-6-g49f6