diff options
Diffstat (limited to 'ci')
-rwxr-xr-x | ci/check-directional-formatting.bash | 27 | ||||
-rw-r--r-- | ci/config/README | 14 | ||||
-rwxr-xr-x | ci/config/allow-ref.sample | 27 | ||||
-rwxr-xr-x | ci/install-dependencies.sh | 68 | ||||
-rwxr-xr-x | ci/install-docker-dependencies.sh | 34 | ||||
-rwxr-xr-x | ci/lib.sh | 292 | ||||
-rwxr-xr-x | ci/make-test-artifacts.sh | 2 | ||||
-rwxr-xr-x | ci/print-test-failures.sh | 20 | ||||
-rwxr-xr-x | ci/run-build-and-minimal-fuzzers.sh | 19 | ||||
-rwxr-xr-x | ci/run-build-and-tests.sh | 42 | ||||
-rwxr-xr-x | ci/run-docker-build.sh | 11 | ||||
-rwxr-xr-x | ci/run-docker.sh | 4 | ||||
-rwxr-xr-x | ci/run-static-analysis.sh | 2 | ||||
-rwxr-xr-x | ci/run-test-slice.sh | 10 |
14 files changed, 403 insertions, 169 deletions
diff --git a/ci/check-directional-formatting.bash b/ci/check-directional-formatting.bash new file mode 100755 index 0000000..e6211b1 --- /dev/null +++ b/ci/check-directional-formatting.bash @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script verifies that the non-binary files tracked in the Git index do +# not contain any Unicode directional formatting: such formatting could be used +# to deceive reviewers into interpreting code differently from the compiler. +# This is intended to run on an Ubuntu agent in a GitHub workflow. +# +# To allow translated messages to introduce such directional formatting in the +# future, we exclude the `.po` files from this validation. +# +# Neither GNU grep nor `git grep` (not even with `-P`) handle `\u` as a way to +# specify UTF-8. +# +# To work around that, we use `printf` to produce the pattern as a byte +# sequence, and then feed that to `git grep` as a byte sequence (setting +# `LC_CTYPE` to make sure that the arguments are interpreted as intended). +# +# Note: we need to use Bash here because its `printf` interprets `\uNNNN` as +# UTF-8 code points, as desired. Running this script through Ubuntu's `dash`, +# for example, would use a `printf` that does not understand that syntax. + +# U+202a..U+2a2e: LRE, RLE, PDF, LRO and RLO +# U+2066..U+2069: LRI, RLI, FSI and PDI +regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)' + +! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \ + -- ':(exclude,attr:binary)' ':(exclude)*.po' diff --git a/ci/config/README b/ci/config/README new file mode 100644 index 0000000..8de3a04 --- /dev/null +++ b/ci/config/README @@ -0,0 +1,14 @@ +You can configure some aspects of the GitHub Actions-based CI on a +per-repository basis by setting "variables" and "secrets" from with the +GitHub web interface. These can be found at: + + https://github.com/<user>/git/settings/secrets/actions + +The following variables can be used: + + - CI_BRANCHES + + By default, CI is run when any branch is pushed. If this variable is + non-empty, then only the branches it lists will run CI. Branch names + should be separated by spaces, and should use their shortened form + (e.g., "main", not "refs/heads/main"). diff --git a/ci/config/allow-ref.sample b/ci/config/allow-ref.sample deleted file mode 100755 index af0e076..0000000 --- a/ci/config/allow-ref.sample +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# -# Sample script for enabling/disabling GitHub Actions CI runs on -# particular refs. By default, CI is run for all branches pushed to -# GitHub. You can override this by dropping the ".sample" from the script, -# editing it, committing, and pushing the result to the "ci-config" branch of -# your repository: -# -# git checkout -b ci-config -# cp allow-ref.sample allow-ref -# $EDITOR allow-ref -# git add allow-ref -# git commit -am "implement my ci preferences" -# git push -# -# This script will then be run when any refs are pushed to that repository. It -# gets the fully qualified refname as the first argument, and should exit with -# success only for refs for which you want to run CI. - -case "$1" in -# allow one-off tests by pushing to "for-ci" or "for-ci/mybranch" -refs/heads/for-ci*) true ;; -# always build your integration branch -refs/heads/my-integration-branch) true ;; -# don't build any other branches or tags -*) false ;; -esac diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 0229a77..b4e22de 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -5,24 +5,17 @@ . ${0%/*}/lib.sh -P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION +P4WHENCE=https://cdist2.perforce.com/perforce/r21.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" -case "$jobname" in -linux-clang|linux-gcc) - sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" +case "$runs_on_pool" in +ubuntu-*) sudo apt-get -q update sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ - $UBUNTU_COMMON_PKGS - case "$jobname" in - linux-gcc) - sudo apt-get -q -y install gcc-8 - ;; - esac - + $UBUNTU_COMMON_PKGS $CC_PACKAGE $PYTHON_PACKAGE mkdir --parents "$P4_PATH" pushd "$P4_PATH" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" @@ -37,34 +30,41 @@ linux-clang|linux-gcc) cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . popd ;; -osx-clang|osx-gcc) +macos-*) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 # Uncomment this if you want to run perf tests: # brew install gnu-time test -z "$BREW_INSTALL_PACKAGES" || brew install $BREW_INSTALL_PACKAGES brew link --force gettext - brew cask install --no-quarantine perforce || { - # Update the definitions and try again - cask_repo="$(brew --repository)"/Library/Taps/homebrew/homebrew-cask && - git -C "$cask_repo" pull --no-stat && - brew cask install --no-quarantine perforce - } || - brew install caskroom/cask/perforce - case "$jobname" in - osx-gcc) - brew install gcc@9 - # Just in case the image is updated to contain gcc@9 - # pre-installed but not linked. - brew link gcc@9 - ;; - esac + + mkdir -p "$P4_PATH" + pushd "$P4_PATH" + wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && + tar -xf helix-core-server.tgz && + sudo xattr -d com.apple.quarantine p4 p4d 2>/dev/null || true + popd + + if test -n "$CC_PACKAGE" + then + BREW_PACKAGE=${CC_PACKAGE/-/@} + brew install "$BREW_PACKAGE" + brew link "$BREW_PACKAGE" + fi ;; +esac + +case "$jobname" in StaticAnalysis) sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext make ;; +sparse) + sudo apt-get -q update -q + sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \ + libexpat-dev gettext zlib1g-dev + ;; Documentation) sudo apt-get -q update sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make @@ -72,21 +72,25 @@ Documentation) test -n "$ALREADY_HAVE_ASCIIDOCTOR" || sudo gem install --version 1.5.8 asciidoctor ;; -linux-gcc-4.8|GETTEXT_POISON) +linux-gcc-default) sudo apt-get -q update sudo apt-get -q -y install $UBUNTU_COMMON_PKGS ;; esac -if type p4d >/dev/null && type p4 >/dev/null +if type p4d >/dev/null 2>&1 && type p4 >/dev/null 2>&1 then echo "$(tput setaf 6)Perforce Server Version$(tput sgr0)" - p4d -V | grep Rev. + p4d -V echo "$(tput setaf 6)Perforce Client Version$(tput sgr0)" - p4 -V | grep Rev. + p4 -V +else + echo >&2 "WARNING: perforce wasn't installed, see above for clues why" fi -if type git-lfs >/dev/null +if type git-lfs >/dev/null 2>&1 then echo "$(tput setaf 6)Git-LFS Version$(tput sgr0)" git-lfs version +else + echo >&2 "WARNING: git-lfs wasn't installed, see above for clues why" fi diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index 26a6689..eb2c9e1 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -3,8 +3,12 @@ # Install dependencies required to build and test Git inside container # +. ${0%/*}/lib.sh + +begin_group "Install dependencies" + case "$jobname" in -Linux32) +linux32) linux32 --32bit i386 sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev \ @@ -12,7 +16,31 @@ Linux32) ' ;; linux-musl) - apk add --update build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null + ;; +linux-*|StaticAnalysis) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + + apt update -q && + apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \ + libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \ + perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ + libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ + apache2 cvs cvsps gnupg libcgi-pm-perl subversion + + if test "$jobname" = StaticAnalysis + then + apt install -q -y coccinelle + fi + ;; +pedantic) + dnf -yq update >/dev/null && + dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null ;; esac + +end_group "Install dependencies" @@ -1,5 +1,70 @@ # Library of functions shared by all CI scripts +if test true = "$GITHUB_ACTIONS" +then + begin_group () { + need_to_end_group=t + echo "::group::$1" >&2 + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + echo '::endgroup::' >&2 + } +elif test true = "$GITLAB_CI" +then + begin_group () { + need_to_end_group=t + printf "\e[0Ksection_start:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K$1\n" + trap "end_group '$1'" EXIT + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + printf "\e[0Ksection_end:$(date +%s):$(echo "$1" | tr ' ' _)\r\e[0K\n" + trap - EXIT + } +else + begin_group () { :; } + end_group () { :; } + + set -x +fi + +group () { + group="$1" + shift + begin_group "$group" + + # work around `dash` not supporting `set -o pipefail` + ( + "$@" 2>&1 + echo $? >exit.status + ) | + sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/' + res=$(cat exit.status) + rm exit.status + + end_group "$group" + return $res +} + +begin_group "CI setup" +trap "end_group 'CI setup'" EXIT + +# Set 'exit on error' for all CI scripts to let the caller know that +# something went wrong. +# +# We already enabled tracing executed commands earlier. This helps by showing +# how # environment variables are set and and dependencies are installed. +set -e + skip_branch_tip_with_tag () { # Sometimes, a branch is pushed at the same time the tag that points # at the same commit as the tip of the branch is pushed, and building @@ -20,10 +85,32 @@ skip_branch_tip_with_tag () { fi } +# Check whether we can use the path passed via the first argument as Git +# repository. +is_usable_git_repository () { + # We require Git in our PATH, otherwise we cannot access repositories + # at all. + if ! command -v git >/dev/null + then + return 1 + fi + + # And the target directory needs to be a proper Git repository. + if ! git -C "$1" rev-parse 2>/dev/null + then + return 1 + fi +} + # Save some info about the current commit's tree, so we can skip the build # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { + if ! is_usable_git_repository . + then + return + fi + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp @@ -34,7 +121,12 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if test "$TRAVIS_DEBUG_MODE" = true || test true = "$GITHUB_ACTIONS" + if test true = "$GITHUB_ACTIONS" + then + return + fi + + if ! is_usable_git_repository . then return fi @@ -60,7 +152,7 @@ skip_good_tree () { cat <<-EOF $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. - The log of that build job is available at $(url_for_job_id $prev_good_job_id) + The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id To force a re-build delete the branch's cache and then hit 'Restart job'. EOF fi @@ -69,8 +161,12 @@ skip_good_tree () { exit 0 } -check_unignored_build_artifacts () -{ +check_unignored_build_artifacts () { + if ! is_usable_git_repository . + then + return + fi + ! git ls-files --other --exclude-standard --error-unmatch \ -- ':/*' 2>/dev/null || { @@ -79,41 +175,37 @@ check_unignored_build_artifacts () } } +handle_failed_tests () { + return 1 +} + +create_failed_test_artifacts () { + mkdir -p t/failed-test-artifacts + + for test_exit in t/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the 'print test failures' step below." + echo "See also the 'failed-tests-*' artifacts attached to this run." + cat "t/test-results/$test_name.markup" + + trash_dir="t/trash directory.$test_name" + cp "t/test-results/$test_name.out" t/failed-test-artifacts/ + tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + done +} + # GitHub Action doesn't set TERM, which is required by tput export TERM=${TERM:-dumb} # Clear MAKEFLAGS that may come from the outside world. export MAKEFLAGS= -# Set 'exit on error' for all CI scripts to let the caller know that -# something went wrong. -# Set tracing executed commands, primarily setting environment variables -# and installing dependencies. -set -ex - -if test true = "$TRAVIS" -then - CI_TYPE=travis - # When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not - # what we want here. We want the source branch instead. - CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" - CI_COMMIT="$TRAVIS_COMMIT" - CI_JOB_ID="$TRAVIS_JOB_ID" - CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" - CI_OS_NAME="$TRAVIS_OS_NAME" - CI_REPO_SLUG="$TRAVIS_REPO_SLUG" - - cache_dir="$HOME/travis-cache" - - url_for_job_id () { - echo "https://travis-ci.org/$CI_REPO_SLUG/jobs/$1" - } - - BREW_INSTALL_PACKAGES="git-lfs gettext" - export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --immediate" - MAKEFLAGS="$MAKEFLAGS --jobs=2" -elif test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" +if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" then CI_TYPE=azure-pipelines # We are running in Azure Pipelines @@ -130,15 +222,8 @@ then # among *all* phases) cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" - url_for_job_id () { - echo "$SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$1" - } - - export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" - export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows_nt != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + GIT_TEST_OPTS="--write-junit-xml" + JOBS=10 elif test true = "$GITHUB_ACTIONS" then CI_TYPE=github-actions @@ -148,25 +233,78 @@ then test macos != "$CI_OS_NAME" || CI_OS_NAME=osx CI_REPO_SLUG="$GITHUB_REPOSITORY" CI_JOB_ID="$GITHUB_RUN_ID" - CC="${CC:-gcc}" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV + create_failed_test_artifacts + return 1 + } + + cache_dir="$HOME/none" + + GIT_TEST_OPTS="--github-workflow-markup" + JOBS=10 +elif test true = "$GITLAB_CI" +then + CI_TYPE=gitlab-ci + CI_BRANCH="$CI_COMMIT_REF_NAME" + CI_COMMIT="$CI_COMMIT_SHA" + case "$CI_JOB_IMAGE" in + macos-*) + # GitLab CI has Python installed via multiple package managers, + # most notably via asdf and Homebrew. Ensure that our builds + # pick up the Homebrew one by prepending it to our PATH as the + # asdf one breaks tests. + export PATH="$(brew --prefix)/bin:$PATH" + + CI_OS_NAME=osx + ;; + alpine:*|fedora:*|ubuntu:*) + CI_OS_NAME=linux;; + *) + echo "Could not identify OS image" >&2 + env >&2 + exit 1 + ;; + esac + CI_REPO_SLUG="$CI_PROJECT_PATH" + CI_JOB_ID="$CI_JOB_ID" + CC="${CC_PACKAGE:-${CC:-gcc}}" + DONT_SKIP_TAGS=t + handle_failed_tests () { + create_failed_test_artifacts + return 1 + } cache_dir="$HOME/none" - export GIT_PROVE_OPTS="--timer --jobs 10" - export GIT_TEST_OPTS="--verbose-log -x" - MAKEFLAGS="$MAKEFLAGS --jobs=10" - test windows != "$CI_OS_NAME" || - GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) + JOBS=$(nproc) else echo "Could not identify CI type" >&2 env >&2 exit 1 fi +MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS" +GIT_PROVE_OPTS="--timer --jobs $JOBS" + +GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x" +case "$CI_OS_NAME" in +windows|windows_nt) + GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers" + ;; +esac + +export GIT_TEST_OPTS +export GIT_PROVE_OPTS + good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" +test -n "${DONT_SKIP_TAGS-}" || skip_branch_tip_with_tag skip_good_tree @@ -178,17 +316,22 @@ fi export DEVELOPER=1 export DEFAULT_TEST_TARGET=prove export GIT_TEST_CLONE_2GB=true +export SKIP_DASHED_BUILT_INS=YesPlease -case "$jobname" in -linux-clang|linux-gcc) - if [ "$jobname" = linux-gcc ] +case "$runs_on_pool" in +ubuntu-*) + if test "$jobname" = "linux-gcc-default" then - export CC=gcc-8 - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" - else - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" + break fi + PYTHON_PACKAGE=python2 + if test "$jobname" = linux-gcc + then + PYTHON_PACKAGE=python3 + fi + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE" + export GIT_TEST_HTTPD=true # The Linux build installs the defined dependency versions below. @@ -196,38 +339,47 @@ linux-clang|linux-gcc) # were recorded in the Homebrew database upon creating the OS X # image. # Keep that in mind when you encounter a broken OS X build! - export LINUX_P4_VERSION="16.2" export LINUX_GIT_LFS_VERSION="1.5.2" P4_PATH="$HOME/custom/p4" GIT_LFS_PATH="$HOME/custom/git-lfs" export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" ;; -osx-clang|osx-gcc) - if [ "$jobname" = osx-gcc ] +macos-*) + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" + if [ "$jobname" != osx-gcc ] then - export CC=gcc-9 - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" - else - MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)" + MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" fi - # t9810 occasionally fails on Travis CI OS X - # t9816 occasionally fails with "TAP out of sequence errors" on - # Travis CI OS X - export GIT_SKIP_TESTS="t9810 t9816" - ;; -GETTEXT_POISON) - export GIT_TEST_GETTEXT_POISON=true + P4_PATH="$HOME/custom/p4" + export PATH="$P4_PATH:$PATH" ;; -Linux32) +esac + +case "$jobname" in +linux32) CC=gcc ;; linux-musl) CC=gcc MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" + MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" + ;; +linux-leaks|linux-reftable-leaks) + export SANITIZE=leak + export GIT_TEST_PASSING_SANITIZE_LEAK=true + export GIT_TEST_SANITIZE_LEAK_LOG=true + ;; +linux-asan-ubsan) + export SANITIZE=address,undefined + export NO_SVN_TESTS=LetsSaveSomeTime + MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften" ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" + +end_group "CI setup" +set -x diff --git a/ci/make-test-artifacts.sh b/ci/make-test-artifacts.sh index 6469674..74141af 100755 --- a/ci/make-test-artifacts.sh +++ b/ci/make-test-artifacts.sh @@ -7,6 +7,6 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early . ${0%/*}/lib.sh -make artifacts-tar ARTIFACTS_DIRECTORY="$1" +group Build make artifacts-tar ARTIFACTS_DIRECTORY="$1" check_unignored_build_artifacts diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 92a983a..b1f80ae 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -8,7 +8,7 @@ # Tracing executed commands would produce too much noise in the loop below. set +x -cd t/ +cd "${TEST_OUTPUT_DIRECTORY:-t/}" if ! ls test-results/*.exit >/dev/null 2>/dev/null then @@ -39,8 +39,6 @@ do test_name="${test_name##*/}" trash_dir="trash directory.$test_name" case "$CI_TYPE" in - travis) - ;; azure-pipelines) mkdir -p failed-test-artifacts mv "$trash_dir" failed-test-artifacts @@ -48,7 +46,13 @@ do ;; github-actions) mkdir -p failed-test-artifacts - echo "::set-env name=FAILED_TEST_ARTIFACTS::t/failed-test-artifacts" + echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; + gitlab-ci) + mkdir -p failed-test-artifacts cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" continue @@ -88,11 +92,3 @@ do fi fi done - -if [ $combined_trash_size -gt 0 ] -then - echo "------------------------------------------------------------------------" - echo "Trash directories embedded in this log can be extracted by running:" - echo - echo " curl https://api.travis-ci.org/v3/job/$TRAVIS_JOB_ID/log.txt |./ci/util/extract-trash-dirs.sh" -fi diff --git a/ci/run-build-and-minimal-fuzzers.sh b/ci/run-build-and-minimal-fuzzers.sh new file mode 100755 index 0000000..a51076d --- /dev/null +++ b/ci/run-build-and-minimal-fuzzers.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# +# Build and test Git's fuzzers +# + +. ${0%/*}/lib.sh + +group "Build fuzzers" make \ + CC=clang \ + CXX=clang++ \ + CFLAGS="-fsanitize=fuzzer-no-link,address" \ + LIB_FUZZING_ENGINE="-fsanitize=fuzzer,address" \ + fuzz-all + +for fuzzer in commit-graph config date pack-headers pack-idx ; do + begin_group "fuzz-$fuzzer" + ./oss-fuzz/fuzz-$fuzzer -verbosity=0 -runs=1 || exit 1 + end_group "fuzz-$fuzzer" +done diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 17e25aa..c192bd6 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -10,30 +10,52 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -make +run_tests=t + case "$jobname" in linux-gcc) - make test + export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main + ;; +linux-TEST-vars) export GIT_TEST_SPLIT_INDEX=yes + export GIT_TEST_MERGE_ALGORITHM=recursive export GIT_TEST_FULL_IN_PACK_ARRAY=true export GIT_TEST_OE_SIZE=10 export GIT_TEST_OE_DELTA_SIZE=5 export GIT_TEST_COMMIT_GRAPH=1 export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1 export GIT_TEST_MULTI_PACK_INDEX=1 - export GIT_TEST_ADD_I_USE_BUILTIN=1 - make test + export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1 + export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master + export GIT_TEST_NO_WRITE_REV_INDEX=1 + export GIT_TEST_CHECKOUT_WORKERS=2 + export GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=1 ;; -linux-gcc-4.8) - # Don't run the tests; we only care about whether Git can be - # built with GCC 4.8, as it errors out on some undesired (C99) - # constructs that newer compilers seem to quietly accept. +linux-clang) + export GIT_TEST_DEFAULT_HASH=sha1 ;; -*) - make test +linux-sha256) + export GIT_TEST_DEFAULT_HASH=sha256 + ;; +linux-reftable|linux-reftable-leaks|osx-reftable) + export GIT_TEST_DEFAULT_REF_FORMAT=reftable + ;; +pedantic) + # Don't run the tests; we only care about whether Git can be + # built. + export DEVOPTS=pedantic + run_tests= ;; esac +group Build make +if test -n "$run_tests" +then + group "Run tests" make test || + handle_failed_tests + group "Run unit tests" \ + make DEFAULT_UNIT_TEST_TARGET=unit-tests-prove unit-tests +fi check_unignored_build_artifacts save_good_tree diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index 8d47a5f..6cd832e 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -15,7 +15,7 @@ then fi case "$jobname" in -Linux32) +linux32) switch_cmd="linux32 --32bit i386" ;; linux-musl) @@ -47,15 +47,6 @@ else else useradd -u $HOST_UID $CI_USER fi - - # Due to a bug the test suite was run as root in the past, so - # a prove state file created back then is only accessible by - # root. Now that bug is fixed, the test suite is run as a - # regular user, but the prove state file coming from Travis - # CI's cache might still be owned by root. - # Make sure that this user has rights to any cached files, - # including an existing prove state file. - test -n "$cache_dir" && chown -R $HOST_UID:$HOST_UID "$cache_dir" fi # Build and test diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 37fa372..af89d16 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -6,7 +6,7 @@ . ${0%/*}/lib.sh case "$jobname" in -Linux32) +linux32) CI_CONTAINER="daald/ubuntu32:xenial" ;; linux-musl) @@ -25,7 +25,7 @@ docker pull "$CI_CONTAINER" # root@container:/# export jobname=<jobname> # root@container:/# /usr/src/git/ci/run-docker-build.sh <host-user-id> -container_cache_dir=/tmp/travis-cache +container_cache_dir=/tmp/container-cache docker run \ --interactive \ diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index 65bcebd..0d51e5c 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -29,4 +29,6 @@ fi make hdr-check || exit 1 +make check-pot + save_good_tree diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index f8c2c31..ae80943 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -10,8 +10,14 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -make --quiet -C t T="$(cd t && +group "Run tests" make --quiet -C t T="$(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh | - tr '\n' ' ')" + tr '\n' ' ')" || +handle_failed_tests + +# We only have one unit test at the moment, so run it in the first slice +if [ "$1" == "0" ] ; then + group "Run unit tests" make --quiet -C t unit-tests-prove +fi check_unignored_build_artifacts |