From eafff6e41e6a16c23764dcd31373bedd61572cf1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Feb 2020 13:23:11 +0000 Subject: t/lib-httpd: avoid using macOS' sed Among other differences relative to GNU sed, macOS' sed always ends its output with a trailing newline, even if the input did not have such a trailing newline. Surprisingly, this makes three httpd-based tests fail on macOS: t5616, t5702 and t5703. ("Surprisingly" because those tests have been around for some time, but apparently nobody runs them on macOS with a working Apache2 setup.) The reason is that we use `sed` in those tests to filter the response of the web server. Apart from the fact that we use GNU constructs (such as using a space after the `c` command instead of a backslash and a newline), we have another problem: macOS' sed LF-only newlines while webservers are supposed to use CR/LF ones. Even worse, t5616 uses `sed` to replace a binary part of the response with a new binary part (kind of hoping that the replaced binary part does not contain a 0x0a byte which would be interpreted as a newline). To that end, it calls on Perl to read the binary pack file and hex-encode it, then calls on `sed` to prefix every hex digit pair with a `\x` in order to construct the text that the `c` statement of the `sed` invocation is supposed to insert. So we call Perl and sed to construct a sed statement. The final nail in the coffin is that macOS' sed does not even interpret those `\x` constructs. Let's just replace all of that by Perl snippets. With Perl, at least, we do not have to deal with GNU vs macOS semantics, we do not have to worry about unwanted trailing newlines, and we do not have to spawn commands to construct arguments for other commands to be spawned (i.e. we can avoid a whole lot of shell scripting complexity). The upshot is that this fixes t5616, t5702 and t5703 on macOS with Apache2. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh index 656997b..1449ee9 100644 --- a/t/lib-httpd.sh +++ b/t/lib-httpd.sh @@ -132,7 +132,7 @@ prepare_httpd() { install_script broken-smart-http.sh install_script error-smart-http.sh install_script error.sh - install_script apply-one-time-sed.sh + install_script apply-one-time-perl.sh ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules" diff --git a/t/lib-httpd/apache.conf b/t/lib-httpd/apache.conf index 5c1c86c..994e529 100644 --- a/t/lib-httpd/apache.conf +++ b/t/lib-httpd/apache.conf @@ -113,7 +113,7 @@ Alias /auth/dumb/ www/auth/dumb/ SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH} SetEnv GIT_HTTP_EXPORT_ALL - + SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH} SetEnv GIT_HTTP_EXPORT_ALL @@ -122,7 +122,7 @@ ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1 ScriptAlias /broken_smart/ broken-smart-http.sh/ ScriptAlias /error_smart/ error-smart-http.sh/ ScriptAlias /error/ error.sh/ -ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1 +ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1 Options FollowSymlinks @@ -135,7 +135,7 @@ ScriptAliasMatch /one_time_sed/(.*) apply-one-time-sed.sh/$1 Options ExecCGI - + Options ExecCGI diff --git a/t/lib-httpd/apply-one-time-perl.sh b/t/lib-httpd/apply-one-time-perl.sh new file mode 100644 index 0000000..09a0abd --- /dev/null +++ b/t/lib-httpd/apply-one-time-perl.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# If "one-time-perl" exists in $HTTPD_ROOT_PATH, run perl on the HTTP response, +# using the contents of "one-time-perl" as the perl command to be run. If the +# response was modified as a result, delete "one-time-perl" so that subsequent +# HTTP responses are no longer modified. +# +# This can be used to simulate the effects of the repository changing in +# between HTTP request-response pairs. +if test -f one-time-perl +then + LC_ALL=C + export LC_ALL + + "$GIT_EXEC_PATH/git-http-backend" >out + perl -pe "$(cat one-time-perl)" out >out_modified + + if cmp -s out out_modified + then + cat out + else + cat out_modified + rm one-time-perl + fi +else + "$GIT_EXEC_PATH/git-http-backend" +fi diff --git a/t/lib-httpd/apply-one-time-sed.sh b/t/lib-httpd/apply-one-time-sed.sh deleted file mode 100644 index bf7689d..0000000 --- a/t/lib-httpd/apply-one-time-sed.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -# If "one-time-sed" exists in $HTTPD_ROOT_PATH, run sed on the HTTP response, -# using the contents of "one-time-sed" as the sed command to be run. If the -# response was modified as a result, delete "one-time-sed" so that subsequent -# HTTP responses are no longer modified. -# -# This can be used to simulate the effects of the repository changing in -# between HTTP request-response pairs. -if test -f one-time-sed -then - "$GIT_EXEC_PATH/git-http-backend" >out - sed "$(cat one-time-sed)" out >out_modified - - if cmp -s out out_modified - then - cat out - else - cat out_modified - rm one-time-sed - fi -else - "$GIT_EXEC_PATH/git-http-backend" -fi diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 97a6772..dd9cd4b 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -233,7 +233,7 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f git -C "$REPO" config protocol.version 2 && git -C client config protocol.version 2 && - git -C client fetch --depth=2 "$HTTPD_URL/one_time_sed/repo" master:a_branch && + git -C client fetch --depth=2 "$HTTPD_URL/one_time_perl/repo" master:a_branch && # Craft a situation in which the server sends back an unshallow request # with an empty packfile. This is done by refetching with a shorter @@ -242,13 +242,13 @@ test_expect_success 'shallow fetches check connectivity before writing shallow f printf "s/0034shallow %s/0036unshallow %s/" \ "$(git -C "$REPO" rev-parse HEAD)" \ "$(git -C "$REPO" rev-parse HEAD^)" \ - >"$HTTPD_ROOT_PATH/one-time-sed" && + >"$HTTPD_ROOT_PATH/one-time-perl" && test_must_fail env GIT_TEST_SIDEBAND_ALL=0 git -C client \ - fetch --depth=1 "$HTTPD_URL/one_time_sed/repo" \ + fetch --depth=1 "$HTTPD_URL/one_time_perl/repo" \ master:a_branch && - # Ensure that the one-time-sed script was used. - ! test -e "$HTTPD_ROOT_PATH/one-time-sed" && + # Ensure that the one-time-perl script was used. + ! test -e "$HTTPD_ROOT_PATH/one-time-perl" && # Ensure that the resulting repo is consistent, despite our failure to # fetch. diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 9a9178f..0eb5b1c4 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -398,14 +398,18 @@ intersperse () { sed 's/\(..\)/'$1'\1/g' } -# Create a one-time-sed command to replace the existing packfile with $1. +# Create a one-time-perl command to replace the existing packfile with $1. replace_packfile () { # The protocol requires that the packfile be sent in sideband 1, hence # the extra \x01 byte at the beginning. - printf "1,/packfile/!c %04x\\\\x01%s0000" \ - "$(($(wc -c <$1) + 5))" \ - "$(hex_unpack <$1 | intersperse '\\x')" \ - >"$HTTPD_ROOT_PATH/one-time-sed" + cp $1 "$HTTPD_ROOT_PATH/one-time-pack" && + echo 'if (/packfile/) { + print; + my $length = -s "one-time-pack"; + printf "%04x\x01", $length + 5; + print `cat one-time-pack` . "0000"; + last + }' >"$HTTPD_ROOT_PATH/one-time-perl" } test_expect_success 'upon cloning, check that all refs point to objects' ' @@ -429,16 +433,16 @@ test_expect_success 'upon cloning, check that all refs point to objects' ' # \x01 byte at the beginning. replace_packfile incomplete.pack && - # Use protocol v2 because the sed command looks for the "packfile" + # Use protocol v2 because the perl command looks for the "packfile" # section header. test_config -C "$SERVER" protocol.version 2 && test_must_fail git -c protocol.version=2 clone \ - --filter=blob:none $HTTPD_URL/one_time_sed/server repo 2>err && + --filter=blob:none $HTTPD_URL/one_time_perl/server repo 2>err && test_i18ngrep "did not send all necessary objects" err && - # Ensure that the one-time-sed script was used. - ! test -e "$HTTPD_ROOT_PATH/one-time-sed" + # Ensure that the one-time-perl script was used. + ! test -e "$HTTPD_ROOT_PATH/one-time-perl" ' test_expect_success 'when partial cloning, tolerate server not sending target of tag' ' @@ -469,17 +473,17 @@ test_expect_success 'when partial cloning, tolerate server not sending target of # \x01 byte at the beginning. replace_packfile incomplete.pack && - # Use protocol v2 because the sed command looks for the "packfile" + # Use protocol v2 because the perl command looks for the "packfile" # section header. test_config -C "$SERVER" protocol.version 2 && # Exercise to make sure it works. git -c protocol.version=2 clone \ - --filter=blob:none $HTTPD_URL/one_time_sed/server repo 2> err && + --filter=blob:none $HTTPD_URL/one_time_perl/server repo 2> err && ! grep "missing object referenced by" err && - # Ensure that the one-time-sed script was used. - ! test -e "$HTTPD_ROOT_PATH/one-time-sed" + # Ensure that the one-time-perl script was used. + ! test -e "$HTTPD_ROOT_PATH/one-time-perl" ' test_expect_success 'tolerate server sending REF_DELTA against missing promisor objects' ' @@ -502,7 +506,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor # Clone. The client has deltabase_have but not deltabase_missing. git -c protocol.version=2 clone --no-checkout \ - --filter=blob:none $HTTPD_URL/one_time_sed/server repo && + --filter=blob:none $HTTPD_URL/one_time_perl/server repo && git -C repo hash-object -w -- "$SERVER/have.txt" && # Sanity check to ensure that the client does not have @@ -543,7 +547,7 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor replace_packfile thin.pack && - # Use protocol v2 because the sed command looks for the "packfile" + # Use protocol v2 because the perl command looks for the "packfile" # section header. test_config -C "$SERVER" protocol.version 2 && @@ -556,8 +560,8 @@ test_expect_success 'tolerate server sending REF_DELTA against missing promisor grep "want $(cat deltabase_missing)" trace && ! grep "want $(cat deltabase_have)" trace && - # Ensure that the one-time-sed script was used. - ! test -e "$HTTPD_ROOT_PATH/one-time-sed" + # Ensure that the one-time-perl script was used. + ! test -e "$HTTPD_ROOT_PATH/one-time-perl" ' # DO NOT add non-httpd-specific tests here, because the last part of this diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 7fd7102..5039e66 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -712,11 +712,11 @@ test_expect_success 'when server sends "ready", expect DELIM' ' # After "ready" in the acknowledgments section, pretend that a FLUSH # (0000) was sent instead of a DELIM (0001). - printf "/ready/,$ s/0001/0000/" \ - >"$HTTPD_ROOT_PATH/one-time-sed" && + printf "\$ready = 1 if /ready/; \$ready && s/0001/0000/" \ + >"$HTTPD_ROOT_PATH/one-time-perl" && test_must_fail git -C http_child -c protocol.version=2 \ - fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err && + fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err && test_i18ngrep "expected packfile to be sent after .ready." err ' @@ -737,12 +737,12 @@ test_expect_success 'when server does not send "ready", expect FLUSH' ' # After the acknowledgments section, pretend that a DELIM # (0001) was sent instead of a FLUSH (0000). - printf "/acknowledgments/,$ s/0000/0001/" \ - >"$HTTPD_ROOT_PATH/one-time-sed" && + printf "\$ack = 1 if /acknowledgments/; \$ack && s/0000/0001/" \ + >"$HTTPD_ROOT_PATH/one-time-perl" && test_must_fail env GIT_TRACE_PACKET="$(pwd)/log" git -C http_child \ -c protocol.version=2 \ - fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err && + fetch "$HTTPD_URL/one_time_perl/http_parent" 2> err && grep "fetch< .*acknowledgments" log && ! grep "fetch< .*ready" log && test_i18ngrep "expected no other sections to be sent after no .ready." err diff --git a/t/t5703-upload-pack-ref-in-want.sh b/t/t5703-upload-pack-ref-in-want.sh index 1424fab..e1eec7d 100755 --- a/t/t5703-upload-pack-ref-in-want.sh +++ b/t/t5703-upload-pack-ref-in-want.sh @@ -313,7 +313,7 @@ test_expect_success 'setup repos for change-while-negotiating test' ' test_commit m3 && git tag -d m2 m3 ) && - git -C "$LOCAL_PRISTINE" remote set-url origin "http://127.0.0.1:$LIB_HTTPD_PORT/one_time_sed/repo" && + git -C "$LOCAL_PRISTINE" remote set-url origin "http://127.0.0.1:$LIB_HTTPD_PORT/one_time_perl/repo" && git -C "$LOCAL_PRISTINE" config protocol.version 2 ' @@ -326,7 +326,7 @@ inconsistency () { # RPCs during a single negotiation. oid1=$(git -C "$REPO" rev-parse $1) && oid2=$(git -C "$REPO" rev-parse $2) && - echo "s/$oid1/$oid2/" >"$HTTPD_ROOT_PATH/one-time-sed" + echo "s/$oid1/$oid2/" >"$HTTPD_ROOT_PATH/one-time-perl" } test_expect_success 'server is initially ahead - no ref in want' ' @@ -378,7 +378,7 @@ test_expect_success 'server loses a ref - ref in want' ' git -C "$REPO" config uploadpack.allowRefInWant true && rm -rf local && cp -r "$LOCAL_PRISTINE" local && - echo "s/master/raster/" >"$HTTPD_ROOT_PATH/one-time-sed" && + echo "s/master/raster/" >"$HTTPD_ROOT_PATH/one-time-perl" && test_must_fail git -C local fetch 2>err && test_i18ngrep "fatal: remote error: unknown ref refs/heads/raster" err -- cgit v0.10.2-6-g49f6 From 5ed9fc3fc8629d9c13b6bae03a32e28cf8be3e96 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Feb 2020 13:23:12 +0000 Subject: ci: prevent `perforce` from being quarantined The most recent Azure Pipelines macOS agents enable what Apple calls "System Integrity Protection". This makes `p4d -V` hang: there is some sort of GUI dialog waiting for the user to acknowledge that the copied binaries are legit and may be executed, but on build agents, there is no user who could acknowledge that. Let's ask Homebrew specifically to _not_ quarantine the Perforce binaries. Helped-by: Aleksandr Chebotov Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index cd59855..497fd32 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -40,11 +40,11 @@ osx-clang|osx-gcc) test -z "$BREW_INSTALL_PACKAGES" || brew install $BREW_INSTALL_PACKAGES brew link --force gettext - brew cask install perforce || { + brew cask install --no-quarantine perforce || { # Update the definitions and try again cask_repo="$(brew --repository)"/Library/Taps/homebrew/homebrew-cask && git -C "$cask_repo" pull --no-stat && - brew cask install perforce + brew cask install --no-quarantine perforce } || brew install caskroom/cask/perforce case "$jobname" in -- cgit v0.10.2-6-g49f6 From 7f487ce062cb09f5e58ca0594396ef66a16934e2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Feb 2020 13:23:13 +0000 Subject: Azure Pipeline: switch to the latest agent pools It would seem that at least the `vs2015-win2012r2` pool (which we use via its old name, `Hosted`) is about to be phased out. Let's switch before that. While at it, use the newer pool names as suggested at https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops#use-a-microsoft-hosted-agent Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano diff --git a/azure-pipelines.yml b/azure-pipelines.yml index af2a5ea..675c3a4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,7 +5,8 @@ jobs: - job: windows_build displayName: Windows Build condition: succeeded() - pool: Hosted + pool: + vmImage: windows-latest timeoutInMinutes: 240 steps: - powershell: | @@ -61,7 +62,8 @@ jobs: displayName: Windows Test dependsOn: windows_build condition: succeeded() - pool: Hosted + pool: + vmImage: windows-latest timeoutInMinutes: 240 strategy: parallel: 10 @@ -133,7 +135,8 @@ jobs: - job: vs_build displayName: Visual Studio Build condition: succeeded() - pool: Hosted VS2017 + pool: + vmImage: windows-latest timeoutInMinutes: 240 steps: - powershell: | @@ -181,6 +184,7 @@ jobs: platform: x64 configuration: Release maximumCpuCount: 4 + msbuildArguments: /p:PlatformToolset=v142 - powershell: | & compat\vcbuild\vcpkg_copy_dlls.bat release if (!$?) { exit(1) } @@ -224,7 +228,8 @@ jobs: displayName: Visual Studio Test dependsOn: vs_build condition: succeeded() - pool: Hosted + pool: + vmImage: windows-latest timeoutInMinutes: 240 strategy: parallel: 10 @@ -292,7 +297,8 @@ jobs: - job: linux_clang displayName: linux-clang condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -330,7 +336,8 @@ jobs: - job: linux_gcc displayName: linux-gcc condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -367,7 +374,8 @@ jobs: - job: osx_clang displayName: osx-clang condition: succeeded() - pool: Hosted macOS + pool: + vmImage: macOS-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -402,7 +410,8 @@ jobs: - job: osx_gcc displayName: osx-gcc condition: succeeded() - pool: Hosted macOS + pool: + vmImage: macOS-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -435,7 +444,8 @@ jobs: - job: gettext_poison displayName: GETTEXT_POISON condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -472,7 +482,8 @@ jobs: - job: linux32 displayName: Linux32 condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -506,7 +517,8 @@ jobs: - job: static_analysis displayName: StaticAnalysis condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 @@ -526,7 +538,8 @@ jobs: - job: documentation displayName: Documentation condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-latest steps: - bash: | test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 -- cgit v0.10.2-6-g49f6