From b4f71cd47705ab65811addd13bf19d448664abf3 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 7 Jan 2020 11:08:58 +0100 Subject: [PATCH 01/10] ci: fix azure pipeline specific code in install-awscli --- src/ci/scripts/install-awscli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh index e2118793850..f9b759fe343 100755 --- a/src/ci/scripts/install-awscli.sh +++ b/src/ci/scripts/install-awscli.sh @@ -28,7 +28,7 @@ if isLinux; then pipflags="--user" sudo apt-get install -y python3-setuptools - echo "##vso[task.prependpath]$HOME/.local/bin" + ciCommandAddPath "${HOME}/.local/bin" fi mkdir -p "${DEPS_DIR}" From 9d5c416037b2066b0b1450952914989dee73900a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 16:49:39 +0100 Subject: [PATCH 02/10] ci: rename script windows-symlink-build-dir to symlink-build-dir There will be a need to symlink the Linux build directory in the future as well, so let's make the script name generic. --- src/ci/azure-pipelines/steps/run.yml | 4 ++-- .../{windows-symlink-build-dir.sh => symlink-build-dir.sh} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename src/ci/scripts/{windows-symlink-build-dir.sh => symlink-build-dir.sh} (100%) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index ee9425aa1c5..e2f4b62deaf 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -59,8 +59,8 @@ steps: displayName: Install InnoSetup condition: and(succeeded(), not(variables.SKIP_JOB)) -- bash: src/ci/scripts/windows-symlink-build-dir.sh - displayName: Ensure the build happens on C:\ instead of D:\ +- bash: src/ci/scripts/symlink-build-dir.sh + displayName: Ensure the build happens on a partition with enough space condition: and(succeeded(), not(variables.SKIP_JOB)) - bash: src/ci/scripts/disable-git-crlf-conversion.sh diff --git a/src/ci/scripts/windows-symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh similarity index 100% rename from src/ci/scripts/windows-symlink-build-dir.sh rename to src/ci/scripts/symlink-build-dir.sh From 9beb8f54774ca0d41dd2eb7622809f4073676757 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 26 Nov 2019 12:06:30 +0100 Subject: [PATCH 03/10] ci: add github actions configuration --- .github/workflows/ci.yml | 781 ++++++++++++++++++++++ Cargo.lock | 56 +- Cargo.toml | 1 + src/bootstrap/builder.rs | 5 + src/bootstrap/flags.rs | 24 + src/bootstrap/lib.rs | 1 + src/bootstrap/run.rs | 45 ++ src/bootstrap/test.rs | 29 + src/bootstrap/tool.rs | 1 + src/ci/docker/mingw-check/Dockerfile | 3 +- src/ci/exec-with-shell.py | 16 + src/ci/github-actions/ci.yml | 707 ++++++++++++++++++++ src/ci/scripts/install-mingw.sh | 4 +- src/ci/scripts/install-msys2.sh | 3 + src/ci/scripts/setup-environment.sh | 34 +- src/tools/expand-yaml-anchors/Cargo.toml | 9 + src/tools/expand-yaml-anchors/src/main.rs | 202 ++++++ 17 files changed, 1909 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 src/bootstrap/run.rs create mode 100755 src/ci/exec-with-shell.py create mode 100644 src/ci/github-actions/ci.yml create mode 100644 src/tools/expand-yaml-anchors/Cargo.toml create mode 100644 src/tools/expand-yaml-anchors/src/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..275823e9dbc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,781 @@ +############################################################# +# WARNING: automatically generated file, DO NOT CHANGE! # +############################################################# + +# This file was automatically generated by the expand-yaml-anchors tool. The +# source file that generated this one is: +# +# src/ci/github-actions/ci.yml +# +# Once you make changes to that file you need to run: +# +# ./x.py run src/tools/expand-yaml-anchors/ +# +# The CI build will fail if the tool is not run after changes to this file. + +--- +name: CI +"on": + push: + branches: + - auto + - try + - master + pull_request: + branches: + - "**" +jobs: + pr: + name: PR + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + if: "github.event_name == 'pull_request'" + strategy: + matrix: + name: + - mingw-check + - x86_64-gnu-llvm-7 + - x86_64-gnu-tools + include: + - name: mingw-check + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-llvm-7 + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-tools + env: + CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 + os: ubuntu-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + try: + name: try + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/try'" + strategy: + matrix: + name: + - dist-x86_64-linux + - dist-x86_64-linux-alt + include: + - name: dist-x86_64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + os: ubuntu-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + auto: + name: auto + env: + CI_JOB_NAME: "${{ matrix.name }}" + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/auto'" + strategy: + matrix: + name: + - arm-android + - armhf-gnu + - dist-aarch64-linux + - dist-android + - dist-arm-linux + - dist-armhf-linux + - dist-armv7-linux + - dist-i586-gnu-i586-i686-musl + - dist-i686-freebsd + - dist-i686-linux + - dist-i686-mingw + - dist-i686-msvc + - dist-mips-linux + - dist-mips64-linux + - dist-mips64el-linux + - dist-mipsel-linux + - dist-powerpc-linux + - dist-powerpc64-linux + - dist-powerpc64le-linux + - dist-s390x-linux + - dist-various-1 + - dist-various-2 + - dist-x86_64-apple + - dist-x86_64-apple-alt + - dist-x86_64-freebsd + - dist-x86_64-linux + - dist-x86_64-linux-alt + - dist-x86_64-mingw + - dist-x86_64-msvc + - dist-x86_64-msvc-alt + - dist-x86_64-musl + - dist-x86_64-netbsd + - i686-gnu + - i686-gnu-nopt + - i686-mingw-1 + - i686-mingw-2 + - i686-msvc-1 + - i686-msvc-2 + - mingw-check + - test-various + - wasm32 + - x86_64-apple + - x86_64-gnu + - x86_64-gnu-aux + - x86_64-gnu-debug + - x86_64-gnu-distcheck + - x86_64-gnu-full-bootstrap + - x86_64-gnu-llvm-7 + - x86_64-gnu-nopt + - x86_64-gnu-tools + - x86_64-mingw-1 + - x86_64-mingw-2 + - x86_64-msvc-1 + - x86_64-msvc-2 + - x86_64-msvc-aux + - x86_64-msvc-cargo + - x86_64-msvc-tools + include: + - name: arm-android + os: ubuntu-latest-xl + env: {} + - name: armhf-gnu + os: ubuntu-latest-xl + env: {} + - name: dist-aarch64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-android + os: ubuntu-latest-xl + env: {} + - name: dist-arm-linux + os: ubuntu-latest-xl + env: {} + - name: dist-armhf-linux + os: ubuntu-latest-xl + env: {} + - name: dist-armv7-linux + os: ubuntu-latest-xl + env: {} + - name: dist-i586-gnu-i586-i686-musl + os: ubuntu-latest-xl + env: {} + - name: dist-i686-freebsd + os: ubuntu-latest-xl + env: {} + - name: dist-i686-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mips64el-linux + os: ubuntu-latest-xl + env: {} + - name: dist-mipsel-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-powerpc64le-linux + os: ubuntu-latest-xl + env: {} + - name: dist-s390x-linux + os: ubuntu-latest-xl + env: {} + - name: dist-various-1 + os: ubuntu-latest-xl + env: {} + - name: dist-various-2 + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-freebsd + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + os: ubuntu-latest-xl + - name: dist-x86_64-musl + os: ubuntu-latest-xl + env: {} + - name: dist-x86_64-netbsd + os: ubuntu-latest-xl + env: {} + - name: i686-gnu + os: ubuntu-latest-xl + env: {} + - name: i686-gnu-nopt + os: ubuntu-latest-xl + env: {} + - name: mingw-check + os: ubuntu-latest-xl + env: {} + - name: test-various + os: ubuntu-latest-xl + env: {} + - name: wasm32 + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-aux + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-debug + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-distcheck + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-full-bootstrap + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-llvm-7 + env: + RUST_BACKTRACE: 1 + os: ubuntu-latest-xl + - name: x86_64-gnu-nopt + os: ubuntu-latest-xl + env: {} + - name: x86_64-gnu-tools + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + os: ubuntu-latest-xl + - name: dist-x86_64-apple + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: macos-latest + - name: dist-x86_64-apple-alt + env: + SCRIPT: "./x.py dist" + RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + - name: x86_64-apple + env: + SCRIPT: "./x.py test" + RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc" + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + os: macos-latest + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" + SCRIPT: make ci-subset-1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-2 + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" + SCRIPT: make ci-subset-2 + os: windows-latest-xl + - name: i686-msvc-1 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" + SCRIPT: make ci-subset-1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: i686-msvc-2 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" + SCRIPT: make ci-subset-2 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-aux + env: + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc" + os: windows-latest-xl + - name: x86_64-msvc-cargo + env: + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc" + VCVARS_BAT: vcvars64.bat + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-msvc-tools + env: + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json" + os: windows-latest-xl + - name: i686-mingw-1 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" + SCRIPT: make ci-mingw-subset-1 + CUSTOM_MINGW: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: i686-mingw-2 + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" + SCRIPT: make ci-mingw-subset-2 + CUSTOM_MINGW: 1 + os: windows-latest-xl + - name: x86_64-mingw-1 + env: + SCRIPT: make ci-mingw-subset-1 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu" + CUSTOM_MINGW: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + os: windows-latest-xl + - name: x86_64-mingw-2 + env: + SCRIPT: make ci-mingw-subset-2 + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu" + CUSTOM_MINGW: 1 + os: windows-latest-xl + - name: dist-x86_64-msvc + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --target=i586-pc-windows-msvc --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-i686-mingw + env: + RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" + SCRIPT: python x.py dist + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-x86_64-mingw + env: + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + os: windows-latest-xl + - name: dist-x86_64-msvc-alt + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" + SCRIPT: python x.py dist + os: windows-latest-xl + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + if: success() && !env.SKIP_JOB + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install awscli + run: src/ci/scripts/install-awscli.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install sccache + run: src/ci/scripts/install-sccache.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install clang + run: src/ci/scripts/install-clang.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install WIX + run: src/ci/scripts/install-wix.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: install ninja + run: src/ci/scripts/install-ninja.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" + if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" + shell: "python src/ci/exec-with-shell.py {0}" + master: + name: master + runs-on: ubuntu-latest + env: + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate" + TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues" + TOOLSTATE_PUBLISH: 1 + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + if: "github.event_name == 'push' && github.ref == 'refs/heads/master'" + steps: + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + - name: publish toolstate + run: src/ci/publish_toolstate.sh + env: + TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" + shell: "python src/ci/exec-with-shell.py {0}" + if: success() && !env.SKIP_JOB + try-success: + needs: + - try + if: success() + steps: + - name: mark the job as a success + run: exit 0 + name: bors build finished + runs-on: ubuntu-latest + try-failure: + needs: + - try + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + name: bors build finished + runs-on: ubuntu-latest + auto-success: + needs: + - auto + if: success() + steps: + - name: mark the job as a success + run: exit 0 + name: bors build finished + runs-on: ubuntu-latest + auto-failure: + needs: + - auto + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + name: bors build finished + runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 22a06151353..f45637e6479 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -443,7 +443,7 @@ dependencies = [ "textwrap", "unicode-width", "vec_map", - "yaml-rust", + "yaml-rust 0.3.5", ] [[package]] @@ -1026,6 +1026,14 @@ dependencies = [ "walkdir", ] +[[package]] +name = "expand-yaml-anchors" +version = "0.1.0" +dependencies = [ + "yaml-merge-keys", + "yaml-rust 0.4.3", +] + [[package]] name = "failure" version = "0.1.5" @@ -1830,6 +1838,12 @@ dependencies = [ name = "linkchecker" version = "0.1.0" +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" + [[package]] name = "lock_api" version = "0.3.1" @@ -4854,6 +4868,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d" +dependencies = [ + "proc-macro2 1.0.3", + "quote 1.0.2", + "syn 1.0.11", +] + [[package]] name = "thread_local" version = "0.3.6" @@ -5536,8 +5570,28 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yaml-merge-keys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa" +dependencies = [ + "lazy_static 1.4.0", + "thiserror", + "yaml-rust 0.4.3", +] + [[package]] name = "yaml-rust" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" + +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +dependencies = [ + "linked-hash-map", +] diff --git a/Cargo.toml b/Cargo.toml index 2f5a708e8dc..7b5e0fa1c28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "src/tools/miri", "src/tools/rustdoc-themes", "src/tools/unicode-table-generator", + "src/tools/expand-yaml-anchors", ] exclude = [ "build", diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index dd519506d42..243cd3fa199 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -21,6 +21,7 @@ use crate::doc; use crate::flags::Subcommand; use crate::install; use crate::native; +use crate::run; use crate::test; use crate::tool; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir}; @@ -313,6 +314,7 @@ pub enum Kind { Dist, Doc, Install, + Run, } impl<'a> Builder<'a> { @@ -353,6 +355,7 @@ impl<'a> Builder<'a> { } Kind::Test => describe!( crate::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, test::Tidy, test::Ui, test::CompileFail, @@ -454,6 +457,7 @@ impl<'a> Builder<'a> { install::Src, install::Rustc ), + Kind::Run => describe!(run::ExpandYamlAnchors,), } } @@ -507,6 +511,7 @@ impl<'a> Builder<'a> { Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]), Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), + Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), }; diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index d8831c6d9e5..eda26f7df1f 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -86,6 +86,9 @@ pub enum Subcommand { Install { paths: Vec, }, + Run { + paths: Vec, + }, } impl Default for Subcommand { @@ -113,6 +116,7 @@ Subcommands: clean Clean out build directories dist Build distribution artifacts install Install distribution artifacts + run Run tools contained in this repository To learn more about a subcommand, run `./x.py -h`", ); @@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py -h`", || (s == "clean") || (s == "dist") || (s == "install") + || (s == "run") }); let subcommand = match subcommand { Some(s) => s, @@ -400,6 +405,18 @@ Arguments: ./x.py doc --stage 1", ); } + "run" => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + + ./x.py run src/tool/expand-yaml-anchors + + At least a tool needs to be called.", + ); + } _ => {} }; // Get any optional paths which occur after the subcommand @@ -468,6 +485,13 @@ Arguments: "fmt" => Subcommand::Format { check: matches.opt_present("check") }, "dist" => Subcommand::Dist { paths }, "install" => Subcommand::Install { paths }, + "run" => { + if paths.is_empty() { + println!("\nrun requires at least a path!\n"); + usage(1, &opts, &subcommand_help, &extra_help); + } + Subcommand::Run { paths } + } _ => { usage(1, &opts, &subcommand_help, &extra_help); } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a476d25f102..6436fa75655 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -140,6 +140,7 @@ mod format; mod install; mod metadata; mod native; +mod run; mod sanity; mod test; mod tool; diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs new file mode 100644 index 00000000000..22f88b516cd --- /dev/null +++ b/src/bootstrap/run.rs @@ -0,0 +1,45 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::tool::Tool; +use std::process::Command; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` read the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run(cmd); + } + true +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index aa1d1b7c424..e75868f7ebd 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -750,6 +750,35 @@ impl Step for Tidy { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + fn testdir(builder: &Builder<'_>, host: Interned) -> PathBuf { builder.out.join(host).join("test") } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index c8ccba467e5..52f750f448e 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -378,6 +378,7 @@ bootstrap_tool!( RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true; RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile index e973ba2e33c..1293717f975 100644 --- a/src/ci/docker/mingw-check/Dockerfile +++ b/src/ci/docker/mingw-check/Dockerfile @@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh COPY mingw-check/validate-toolstate.sh /scripts/ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 -ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ +ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \ + python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python2.7 ../x.py build --stage 0 src/tools/build-manifest && \ python2.7 ../x.py test --stage 0 src/tools/compiletest && \ python2.7 ../x.py test src/tools/tidy && \ diff --git a/src/ci/exec-with-shell.py b/src/ci/exec-with-shell.py new file mode 100755 index 00000000000..26ce69e33d9 --- /dev/null +++ b/src/ci/exec-with-shell.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# A simple wrapper that forwards the arguments to bash, unless the +# CI_OVERRIDE_SHELL environment variable is present: in that case the content +# of that environment variable is used as the shell path. + +import os +import sys +import subprocess + +try: + shell = os.environ["CI_OVERRIDE_SHELL"] +except KeyError: + shell = "bash" + +res = subprocess.call([shell] + sys.argv[1:]) +sys.exit(res) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml new file mode 100644 index 00000000000..33573a4de9f --- /dev/null +++ b/src/ci/github-actions/ci.yml @@ -0,0 +1,707 @@ +###################################################### +# WARNING! Action needed when changing this file # +###################################################### + +# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the +# CI configuration stored on the repository. To work around that this file is +# expanded by a tool in the repository, and the expansion is committed as well. +# +# After you make any change to the file you'll need to run this command: +# +# ./x.py run src/tools/expand-yaml-anchors +# +# ...and commit the file it updated in addition to this one. If you forget this +# step CI will fail. + +--- + +############################### +# YAML Anchors Definition # +############################### + +# This key contains most of the YAML anchors that will be used later in the +# document. YAML anchors allows us to greatly reduce duplication inside the CI +# configuration by reusing parts of the configuration. +# +# YAML anchors work by defining an anchor with `&anchor-name` and reusing its +# content in another place with `*anchor-name`. The special `<<` map key merges +# the content of the map with the content of the anchor (or list of anchors). +# +# The expand-yaml-anchors tool will automatically remove this block from the +# output YAML file. +x--expand-yaml-anchors--remove: + + - &shared-ci-variables + CI_JOB_NAME: ${{ matrix.name }} + + - &public-variables + SCCACHE_BUCKET: rust-lang-gha-caches + TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate + + - &prod-variables + SCCACHE_BUCKET: rust-lang-gha-caches + DEPLOY_BUCKET: rust-lang-gha + TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate + TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues + TOOLSTATE_PUBLISH: 1 + # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named + # AWS_SECRET_ACCESS_KEY_. Including the key id in the name allows to + # rotate them in a single branch while keeping the old key in another + # branch, which wouldn't be possible if the key was named with the kind + # (caches, artifacts...). + CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 + ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF + + - &base-job + env: {} + + - &job-linux-xl + os: ubuntu-latest-xl + <<: *base-job + + - &job-macos-xl + os: macos-latest # We don't have an XL builder for this + <<: *base-job + + - &job-windows-xl + os: windows-latest-xl + <<: *base-job + + - &step + if: success() && !env.SKIP_JOB + + - &step-run + <<: *step + # While on Linux and macOS builders it just forwards the arguments to the + # system bash, this wrapper allows switching from the host's bash.exe to + # the one we install along with MSYS2 mid-build on Windows. + # + # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL + # environment variable is set pointing to our MSYS2's bash.exe. From that + # moment the host's bash.exe will not be called anymore. + # + # This is needed because we can't launch our own bash.exe from the host + # bash.exe, as that would load two different cygwin1.dll in memory, causing + # "cygwin heap mismatch" errors. + shell: python src/ci/exec-with-shell.py {0} + + - &base-ci-job + timeout-minutes: 600 + runs-on: "${{ matrix.os }}" + env: *shared-ci-variables + steps: + - name: disable git crlf conversion + run: git config --global core.autocrlf false + shell: bash + + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + + - name: configure GitHub Actions to kill the build when outdated + uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master + with: + github_token: "${{ secrets.github_token }}" + <<: *step + + - name: add extra environment variables + run: src/ci/scripts/setup-environment.sh + env: + # Since it's not possible to merge `${{ matrix.env }}` with the other + # variables in `job..env`, the variables defined in the matrix + # are passed to the `setup-environment.sh` script encoded in JSON, + # which then uses log commands to actually set them. + EXTRA_VARIABLES: ${{ toJson(matrix.env) }} + <<: *step-run + + - name: decide whether to skip this job + run: src/ci/scripts/should-skip-this.sh + <<: *step-run + + - name: collect CPU statistics + run: src/ci/scripts/collect-cpu-stats.sh + <<: *step-run + + - name: show the current environment + run: src/ci/scripts/dump-environment.sh + <<: *step-run + + - name: install awscli + run: src/ci/scripts/install-awscli.sh + <<: *step-run + + - name: install sccache + run: src/ci/scripts/install-sccache.sh + <<: *step-run + + - name: install clang + run: src/ci/scripts/install-clang.sh + <<: *step-run + + - name: install WIX + run: src/ci/scripts/install-wix.sh + <<: *step-run + + - name: install InnoSetup + run: src/ci/scripts/install-innosetup.sh + <<: *step-run + + - name: ensure the build happens on a partition with enough space + run: src/ci/scripts/symlink-build-dir.sh + <<: *step-run + + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + <<: *step-run + + - name: install MSYS2 + run: src/ci/scripts/install-msys2.sh + <<: *step-run + + - name: install MSYS2 packages + run: src/ci/scripts/install-msys2-packages.sh + <<: *step-run + + - name: install MinGW + run: src/ci/scripts/install-mingw.sh + <<: *step-run + + - name: install ninja + run: src/ci/scripts/install-ninja.sh + <<: *step-run + + - name: enable ipv6 on Docker + run: src/ci/scripts/enable-docker-ipv6.sh + <<: *step-run + + # Disable automatic line ending conversion (again). On Windows, when we're + # installing dependencies, something switches the git configuration directory or + # re-enables autocrlf. We've not tracked down the exact cause -- and there may + # be multiple -- but this should ensure submodules are checked out with the + # appropriate line endings. + - name: disable git crlf conversion + run: src/ci/scripts/disable-git-crlf-conversion.sh + <<: *step-run + + - name: checkout submodules + run: src/ci/scripts/checkout-submodules.sh + <<: *step-run + + - name: ensure line endings are correct + run: src/ci/scripts/verify-line-endings.sh + <<: *step-run + + - name: run the build + run: src/ci/scripts/run-build-from-ci.sh + env: + AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} + <<: *step-run + + - name: upload artifacts to S3 + run: src/ci/scripts/upload-artifacts.sh + env: + AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} + # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy + # builders *should* have the AWS credentials available. Still, explicitly + # adding the condition is helpful as this way CI will not silently skip + # deploying artifacts from a dist builder if the variables are misconfigured, + # erroring about invalid credentials instead. + if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1') + <<: *step-run + + # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs. + # Check out their documentation for more information on why they're needed. + + - &base-outcome-job + name: bors build finished + runs-on: ubuntu-latest + + - &base-success-job + if: success() + steps: + - name: mark the job as a success + run: exit 0 + <<: *base-outcome-job + + - &base-failure-job + if: "!success()" + steps: + - name: mark the job as a failure + run: exit 1 + <<: *base-outcome-job + +########################### +# Builders definition # +########################### + +name: CI +on: + push: + branches: + - auto + - try + - master + pull_request: + branches: + - "**" + +jobs: + pr: + <<: *base-ci-job + name: PR + env: + <<: [*shared-ci-variables, *public-variables] + if: github.event_name == 'pull_request' + strategy: + matrix: + name: + - mingw-check + - x86_64-gnu-llvm-7 + - x86_64-gnu-tools + include: + - name: mingw-check + <<: *job-linux-xl + + - name: x86_64-gnu-llvm-7 + <<: *job-linux-xl + + - name: x86_64-gnu-tools + env: + CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 + <<: *job-linux-xl + + try: + <<: *base-ci-job + name: try + env: + <<: [*shared-ci-variables, *prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/try' + strategy: + matrix: + name: + - dist-x86_64-linux + - dist-x86_64-linux-alt + include: + - name: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + <<: *job-linux-xl + + auto: + <<: *base-ci-job + name: auto + env: + <<: [*shared-ci-variables, *prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/auto' + strategy: + matrix: + name: + - arm-android + - armhf-gnu + - dist-aarch64-linux + - dist-android + - dist-arm-linux + - dist-armhf-linux + - dist-armv7-linux + - dist-i586-gnu-i586-i686-musl + - dist-i686-freebsd + - dist-i686-linux + - dist-i686-mingw + - dist-i686-msvc + - dist-mips-linux + - dist-mips64-linux + - dist-mips64el-linux + - dist-mipsel-linux + - dist-powerpc-linux + - dist-powerpc64-linux + - dist-powerpc64le-linux + - dist-s390x-linux + - dist-various-1 + - dist-various-2 + - dist-x86_64-apple + - dist-x86_64-apple-alt + - dist-x86_64-freebsd + - dist-x86_64-linux + - dist-x86_64-linux-alt + - dist-x86_64-mingw + - dist-x86_64-msvc + - dist-x86_64-msvc-alt + - dist-x86_64-musl + - dist-x86_64-netbsd + - i686-gnu + - i686-gnu-nopt + - i686-mingw-1 + - i686-mingw-2 + - i686-msvc-1 + - i686-msvc-2 + - mingw-check + - test-various + - wasm32 + - x86_64-apple + - x86_64-gnu + - x86_64-gnu-aux + - x86_64-gnu-debug + - x86_64-gnu-distcheck + - x86_64-gnu-full-bootstrap + - x86_64-gnu-llvm-7 + - x86_64-gnu-nopt + - x86_64-gnu-tools + - x86_64-mingw-1 + - x86_64-mingw-2 + - x86_64-msvc-1 + - x86_64-msvc-2 + - x86_64-msvc-aux + - x86_64-msvc-cargo + - x86_64-msvc-tools + include: + ############################# + # Linux/Docker builders # + ############################# + + - name: arm-android + <<: *job-linux-xl + + - name: armhf-gnu + <<: *job-linux-xl + + - name: dist-aarch64-linux + <<: *job-linux-xl + + - name: dist-android + <<: *job-linux-xl + + - name: dist-arm-linux + <<: *job-linux-xl + + - name: dist-armhf-linux + <<: *job-linux-xl + + - name: dist-armv7-linux + <<: *job-linux-xl + + - name: dist-i586-gnu-i586-i686-musl + <<: *job-linux-xl + + - name: dist-i686-freebsd + <<: *job-linux-xl + + - name: dist-i686-linux + <<: *job-linux-xl + + - name: dist-mips-linux + <<: *job-linux-xl + + - name: dist-mips64-linux + <<: *job-linux-xl + + - name: dist-mips64el-linux + <<: *job-linux-xl + + - name: dist-mipsel-linux + <<: *job-linux-xl + + - name: dist-powerpc-linux + <<: *job-linux-xl + + - name: dist-powerpc64-linux + <<: *job-linux-xl + + - name: dist-powerpc64le-linux + <<: *job-linux-xl + + - name: dist-s390x-linux + <<: *job-linux-xl + + - name: dist-various-1 + <<: *job-linux-xl + + - name: dist-various-2 + <<: *job-linux-xl + + - name: dist-x86_64-freebsd + <<: *job-linux-xl + + - name: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-linux-alt + env: + IMAGE: dist-x86_64-linux + <<: *job-linux-xl + + - name: dist-x86_64-musl + <<: *job-linux-xl + + - name: dist-x86_64-netbsd + <<: *job-linux-xl + + - name: i686-gnu + <<: *job-linux-xl + + - name: i686-gnu-nopt + <<: *job-linux-xl + + - name: mingw-check + <<: *job-linux-xl + + - name: test-various + <<: *job-linux-xl + + - name: wasm32 + <<: *job-linux-xl + + - name: x86_64-gnu + <<: *job-linux-xl + + - name: x86_64-gnu-aux + <<: *job-linux-xl + + - name: x86_64-gnu-debug + <<: *job-linux-xl + + - name: x86_64-gnu-distcheck + <<: *job-linux-xl + + - name: x86_64-gnu-full-bootstrap + <<: *job-linux-xl + + - name: x86_64-gnu-llvm-7 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-xl + + - name: x86_64-gnu-nopt + <<: *job-linux-xl + + - name: x86_64-gnu-tools + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + <<: *job-linux-xl + + #################### + # macOS Builders # + #################### + + - name: dist-x86_64-apple + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-macos-xl + + - name: dist-x86_64-apple-alt + env: + SCRIPT: ./x.py dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + + - name: x86_64-apple + env: + SCRIPT: ./x.py test + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + <<: *job-macos-xl + + ###################### + # Windows Builders # + ###################### + + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-2 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + <<: *job-windows-xl + + - name: i686-msvc-1 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: i686-msvc-2 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-aux + env: + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + <<: *job-windows-xl + + - name: x86_64-msvc-cargo + env: + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-msvc-tools + env: + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json + <<: *job-windows-xl + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + + - name: i686-mingw-1 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-mingw-subset-1 + CUSTOM_MINGW: 1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: i686-mingw-2 + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-mingw-subset-2 + CUSTOM_MINGW: 1 + <<: *job-windows-xl + + - name: x86_64-mingw-1 + env: + SCRIPT: make ci-mingw-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + CUSTOM_MINGW: 1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + <<: *job-windows-xl + + - name: x86_64-mingw-2 + env: + SCRIPT: make ci-mingw-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + CUSTOM_MINGW: 1 + <<: *job-windows-xl + + - name: dist-x86_64-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-i686-msvc + env: + RUST_CONFIGURE_ARGS: >- + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-i686-mingw + env: + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler + SCRIPT: python x.py dist + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-x86_64-mingw + env: + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler + CUSTOM_MINGW: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + <<: *job-windows-xl + + - name: dist-x86_64-msvc-alt + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + <<: *job-windows-xl + + master: + name: master + runs-on: ubuntu-latest + env: + <<: [*prod-variables] + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + steps: + - name: checkout the source code + uses: actions/checkout@v1 + with: + fetch-depth: 2 + + - name: publish toolstate + run: src/ci/publish_toolstate.sh + env: + TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} + <<: *step-run + + # These jobs don't actually test anything, but they're used to tell bors the + # build completed, as there is no practical way to detect when a workflow is + # successful listening to webhooks only. + try-success: + needs: [try] + <<: *base-success-job + try-failure: + needs: [try] + <<: *base-failure-job + auto-success: + needs: [auto] + <<: *base-success-job + auto-failure: + needs: [auto] + <<: *base-failure-job diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 98373df7fce..78728dd7d00 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -50,8 +50,8 @@ if isWindows; then esac if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ - mingw-w64-$arch-gcc mingw-w64-$arch-python2 + pacman -S --noconfirm --needed mingw-w64-$arch-toolchain \ + mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin" else mingw_dir="mingw${bits}" diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index 9e899ba9d89..3c3b5007f86 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -22,4 +22,7 @@ if isWindows; then rm msys2.nupkg chocolatey-core.extension.nupkg mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}" ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin" + + echo "switching shell to use our own bash" + ciCommandSetEnv CI_OVERRIDE_SHELL "$(ciCheckoutPath)/msys2/usr/bin/bash.exe" fi diff --git a/src/ci/scripts/setup-environment.sh b/src/ci/scripts/setup-environment.sh index d134fcd47ba..411ef6f9b28 100755 --- a/src/ci/scripts/setup-environment.sh +++ b/src/ci/scripts/setup-environment.sh @@ -11,16 +11,34 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" # Since matrix variables are readonly in Azure Pipelines, we take # INITIAL_RUST_CONFIGURE_ARGS and establish RUST_CONFIGURE_ARGS # which downstream steps can alter -# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]], -# which was introduced in Bash 4.2 -if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then - INITIAL_RUST_CONFIG="" - echo "No initial Rust configure args set" -else - INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}" - ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}" +if isAzurePipelines; then + # macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]], + # which was introduced in Bash 4.2 + if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then + INITIAL_RUST_CONFIG="" + echo "No initial Rust configure args set" + else + INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}" + ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}" + fi fi +# Load extra environment variables +vars="${EXTRA_VARIABLES-}" +echo "${vars}" | jq '' >/dev/null # Validate JSON and exit on errors +for key in $(echo "${vars}" | jq "keys[]" -r); do + # On Windows, for whatever reason, $key contains the BOM character in it, + # and that messes up `jq ".${key}"`. This line strips the BOM from the key. + # + # https://unix.stackexchange.com/a/381263 + key="$(echo "${key}" | sed '1s/^\xEF\xBB\xBF//')" + + echo "adding extra environment variable ${key}" + value="$(echo "${vars}" | jq ".${key}" -r)" + export "${key}"="${value}" + ciCommandSetEnv "${key}" "${value}" +done + # Builders starting with `dist-` are dist builders, but if they also end with # `-alt` they are alternate dist builders. if [[ "${CI_JOB_NAME}" = dist-* ]]; then diff --git a/src/tools/expand-yaml-anchors/Cargo.toml b/src/tools/expand-yaml-anchors/Cargo.toml new file mode 100644 index 00000000000..2c63e28b693 --- /dev/null +++ b/src/tools/expand-yaml-anchors/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "expand-yaml-anchors" +version = "0.1.0" +authors = ["Pietro Albini "] +edition = "2018" + +[dependencies] +yaml-rust = "0.4.3" +yaml-merge-keys = "0.4.0" diff --git a/src/tools/expand-yaml-anchors/src/main.rs b/src/tools/expand-yaml-anchors/src/main.rs new file mode 100644 index 00000000000..f2ed8aa409a --- /dev/null +++ b/src/tools/expand-yaml-anchors/src/main.rs @@ -0,0 +1,202 @@ +use std::error::Error; +use std::path::{Path, PathBuf}; +use yaml_rust::{Yaml, YamlEmitter, YamlLoader}; + +/// List of directories containing files to expand. The first tuple element is the source +/// directory, while the second tuple element is the destination directory. +#[rustfmt::skip] +static TO_EXPAND: &[(&str, &str)] = &[ + ("src/ci/github-actions", ".github/workflows"), +]; + +/// Name of a special key that will be removed from all the maps in expanded configuration files. +/// This key can then be used to contain shared anchors. +static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove"; + +/// Message that will be included at the top of all the expanded files. {source} will be replaced +/// with the source filename relative to the base path. +static HEADER_MESSAGE: &str = "\ +############################################################# +# WARNING: automatically generated file, DO NOT CHANGE! # +############################################################# + +# This file was automatically generated by the expand-yaml-anchors tool. The +# source file that generated this one is: +# +# {source} +# +# Once you make changes to that file you need to run: +# +# ./x.py run src/tools/expand-yaml-anchors/ +# +# The CI build will fail if the tool is not run after changes to this file. + +"; + +enum Mode { + Check, + Generate, +} + +struct App { + mode: Mode, + base: PathBuf, +} + +impl App { + fn from_args() -> Result> { + // Parse CLI arguments + let args = std::env::args().skip(1).collect::>(); + let (mode, base) = match args.iter().map(|s| s.as_str()).collect::>().as_slice() { + &["generate", ref base] => (Mode::Generate, PathBuf::from(base)), + &["check", ref base] => (Mode::Check, PathBuf::from(base)), + _ => { + eprintln!("usage: expand-yaml-anchors "); + std::process::exit(1); + } + }; + + Ok(App { mode, base }) + } + + fn run(&self) -> Result<(), Box> { + for (source, dest) in TO_EXPAND { + let source = self.base.join(source); + let dest = self.base.join(dest); + for entry in std::fs::read_dir(&source)? { + let path = entry?.path(); + if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") { + continue; + } + + let dest_path = dest.join(path.file_name().unwrap()); + self.expand(&path, &dest_path).with_context(|| match self.mode { + Mode::Generate => format!( + "failed to expand {} into {}", + self.path(&path), + self.path(&dest_path) + ), + Mode::Check => format!("{} is not up to date", self.path(&dest_path)), + })?; + } + } + Ok(()) + } + + fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box> { + let content = std::fs::read_to_string(source) + .with_context(|| format!("failed to read {}", self.path(source)))?; + + let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string()); + + let documents = YamlLoader::load_from_str(&content) + .with_context(|| format!("failed to parse {}", self.path(source)))?; + for mut document in documents.into_iter() { + document = yaml_merge_keys::merge_keys(document) + .with_context(|| format!("failed to expand {}", self.path(source)))?; + document = filter_document(document); + + YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext { + context: "failed to serialize the expanded yaml".into(), + source: Box::new(err), + })?; + buf.push('\n'); + } + + match self.mode { + Mode::Check => { + let old = std::fs::read_to_string(dest) + .with_context(|| format!("failed to read {}", self.path(dest)))?; + if old != buf { + return Err(Box::new(StrError(format!( + "{} and {} are different", + self.path(source), + self.path(dest), + )))); + } + } + Mode::Generate => { + std::fs::write(dest, buf.as_bytes()) + .with_context(|| format!("failed to write to {}", self.path(dest)))?; + } + } + Ok(()) + } + + fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a { + path.strip_prefix(&self.base).unwrap_or(path).display() + } +} + +fn filter_document(document: Yaml) -> Yaml { + match document { + Yaml::Hash(map) => Yaml::Hash( + map.into_iter() + .filter(|(key, _)| { + if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true } + }) + .map(|(key, value)| (filter_document(key), filter_document(value))) + .collect(), + ), + Yaml::Array(vec) => { + Yaml::Array(vec.into_iter().map(|item| filter_document(item)).collect()) + } + other => other, + } +} + +fn main() { + if let Err(err) = App::from_args().and_then(|app| app.run()) { + eprintln!("error: {}", err); + + let mut source = err.as_ref() as &dyn Error; + while let Some(err) = source.source() { + eprintln!("caused by: {}", err); + source = err; + } + + std::process::exit(1); + } +} + +#[derive(Debug)] +struct StrError(String); + +impl Error for StrError {} + +impl std::fmt::Display for StrError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug)] +struct WithContext { + context: String, + source: Box, +} + +impl std::fmt::Display for WithContext { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.context) + } +} + +impl Error for WithContext { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.source.as_ref()) + } +} + +pub(crate) trait ResultExt { + fn with_context String>(self, f: F) -> Result>; +} + +impl>> ResultExt for Result { + fn with_context String>(self, f: F) -> Result> { + match self { + Ok(ok) => Ok(ok), + Err(err) => Err(WithContext { source: err.into(), context: f() }.into()), + } + } +} From b9c3633847cd74f514ebb3197876e8181a003f10 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 16 Dec 2019 12:38:43 +0100 Subject: [PATCH 04/10] ci: don't move the build to C: on GitHub Actions On Azure Pipeliones, the C: filesystem is huge with a lot of free space, while D: is small. By default builds happened in D:, so we added a script to symlink the big directories to C:, granting us more space. Filesystem Size Used Avail Use% C: 256G 143G 114G 56% D: 14G 2.0G 13G 15% On GitHub Actions instead C: is almost full, and we have a lot of free space on D:, where the build happens. Filesystem Size Used Avail Use% C: 128G 114G 15G 89% D: 56G 4.8G 52G 9% This commit stops creating the symlink on GitHub Actions, fixing the out of disk space errors we were seeing on some Windows builders. --- src/ci/scripts/symlink-build-dir.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/scripts/symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh index e57128c70f5..904fd3b60f8 100755 --- a/src/ci/scripts/symlink-build-dir.sh +++ b/src/ci/scripts/symlink-build-dir.sh @@ -9,7 +9,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -if isWindows; then +if isWindows && isAzurePipelines; then cmd //c "mkdir c:\\MORE_SPACE" cmd //c "mklink /J build c:\\MORE_SPACE" fi From 1ba762a193f8db56f4229c6dd3b2a1f0df62170a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 17:02:10 +0100 Subject: [PATCH 05/10] ci: fix out of disk space errors on linux GHA The /mnt mount point has 53GB of free disk space at the time of writing this commit, so this moves the build there to avoid running out of disk space during builds. --- src/ci/scripts/symlink-build-dir.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ci/scripts/symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh index 904fd3b60f8..737a30206af 100755 --- a/src/ci/scripts/symlink-build-dir.sh +++ b/src/ci/scripts/symlink-build-dir.sh @@ -1,8 +1,8 @@ #!/bin/bash -# We've had issues with the default drive in use running out of space during a -# build, and it looks like the `C:` drive has more space than the default `D:` -# drive. We should probably confirm this with the azure pipelines team at some -# point, but this seems to fix our "disk space full" problems. +# We've had multiple issues with the default disk running out of disk space +# during builds, and it looks like other disks mounted in the VMs have more +# space available. This script synlinks the build directory to those other +# disks, in the CI providers and OSes affected by this. set -euo pipefail IFS=$'\n\t' @@ -12,4 +12,8 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows && isAzurePipelines; then cmd //c "mkdir c:\\MORE_SPACE" cmd //c "mklink /J build c:\\MORE_SPACE" +elif isLinux && isGitHubActions; then + sudo mkdir -p /mnt/more-space + sudo chown -R "$(whoami):" /mnt/more-space + ln -s /mnt/more-space obj fi From 11aafa5305656ca3190d099c1bb92b47e587cd0c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 16:46:45 +0100 Subject: [PATCH 06/10] ci: run gha workflows for try, auto and master on the fork --- .github/workflows/ci.yml | 6 +++--- src/ci/github-actions/ci.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275823e9dbc..6fea45a4c91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: TOOLSTATE_PUBLISH: 1 CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF - if: "github.event_name == 'push' && github.ref == 'refs/heads/try'" + if: "github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'" strategy: matrix: name: @@ -301,7 +301,7 @@ jobs: TOOLSTATE_PUBLISH: 1 CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF - if: "github.event_name == 'push' && github.ref == 'refs/heads/auto'" + if: "github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" strategy: matrix: name: @@ -731,7 +731,7 @@ jobs: TOOLSTATE_PUBLISH: 1 CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5 ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF - if: "github.event_name == 'push' && github.ref == 'refs/heads/master'" + if: "github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'" steps: - name: checkout the source code uses: actions/checkout@v1 diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 33573a4de9f..b4f2a367f4d 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -279,7 +279,7 @@ jobs: name: try env: <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && github.ref == 'refs/heads/try' + if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust' strategy: matrix: name: @@ -299,7 +299,7 @@ jobs: name: auto env: <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && github.ref == 'refs/heads/auto' + if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust' strategy: matrix: name: @@ -677,7 +677,7 @@ jobs: runs-on: ubuntu-latest env: <<: [*prod-variables] - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust' steps: - name: checkout the source code uses: actions/checkout@v1 From ba97ad35097146570508dff455f88a194daca3b6 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 18:06:21 +0100 Subject: [PATCH 07/10] ci: move the whole workspace directory on gha/linux Moving just the `obj` directory created problems with mountpoints and Docker containers, so this tries to symlink the parent directory. --- src/ci/scripts/symlink-build-dir.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ci/scripts/symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh index 737a30206af..c77059c00ac 100755 --- a/src/ci/scripts/symlink-build-dir.sh +++ b/src/ci/scripts/symlink-build-dir.sh @@ -15,5 +15,13 @@ if isWindows && isAzurePipelines; then elif isLinux && isGitHubActions; then sudo mkdir -p /mnt/more-space sudo chown -R "$(whoami):" /mnt/more-space - ln -s /mnt/more-space obj + + # Switch the whole workspace to the /mnt partition, which has more space. + # We don't just symlink the `obj` directory as doing that creates problems + # with the docker container. + current_dir="$(readlink -f "$(pwd)")" + cd /tmp + mv "${current_dir}" /mnt/more-space/workspace + ln -s /mnt/more-space/workspace "${current_dir}" + cd "${current_dir}" fi From 89339df264ab09320e31f6edd9a6d355a09244fb Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 18:28:47 +0100 Subject: [PATCH 08/10] ci: run the "bors build finished" jobs only during try and auto runs --- .github/workflows/ci.yml | 8 ++++---- src/ci/github-actions/ci.yml | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fea45a4c91..d1261f04b82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -746,7 +746,7 @@ jobs: try-success: needs: - try - if: success() + if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a success run: exit 0 @@ -755,7 +755,7 @@ jobs: try-failure: needs: - try - if: "!success()" + if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a failure run: exit 1 @@ -764,7 +764,7 @@ jobs: auto-success: needs: - auto - if: success() + if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a success run: exit 0 @@ -773,7 +773,7 @@ jobs: auto-failure: needs: - auto - if: "!success()" + if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" steps: - name: mark the job as a failure run: exit 1 diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index b4f2a367f4d..81a334cd487 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -221,14 +221,12 @@ x--expand-yaml-anchors--remove: runs-on: ubuntu-latest - &base-success-job - if: success() steps: - name: mark the job as a success run: exit 0 <<: *base-outcome-job - &base-failure-job - if: "!success()" steps: - name: mark the job as a failure run: exit 1 @@ -695,13 +693,17 @@ jobs: # successful listening to webhooks only. try-success: needs: [try] + if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job try-failure: needs: [try] + if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job auto-success: needs: [auto] + if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-success-job auto-failure: needs: [auto] + if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" <<: *base-failure-job From f0bd035e2a326c5ee57b74b188675479e365ec67 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 20 Mar 2020 18:37:34 +0100 Subject: [PATCH 09/10] bootstrap: remove default and only_host from expand-yaml-anchors --- src/bootstrap/run.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 22f88b516cd..90053471427 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -7,8 +7,6 @@ pub struct ExpandYamlAnchors; impl Step for ExpandYamlAnchors { type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; /// Runs the `expand-yaml_anchors` tool. /// From 45910e74fde9849feaa34ce851f317bc3c21f14f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 23 Mar 2020 15:01:58 +0100 Subject: [PATCH 10/10] ci: add notice on top of the azure pipelines configuration This will inform contributors tweaking the Azure Pipelines configuration that they also need to tweak the GitHub Actions setup. --- src/ci/azure-pipelines/auto.yml | 11 +++++++++++ src/ci/azure-pipelines/master.yml | 11 +++++++++++ src/ci/azure-pipelines/pr.yml | 11 +++++++++++ src/ci/azure-pipelines/steps/run.yml | 11 +++++++++++ src/ci/azure-pipelines/try.yml | 11 +++++++++++ 5 files changed, 55 insertions(+) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 13db4725ded..56fe3864204 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -1,3 +1,14 @@ +##################################### +## READ BEFORE CHANGING THIS ## +##################################### + +# We're in the process of evaluating GitHub Actions as a possible replacement +# for Azure Pipelines, and at the moment the configuration is duplicated +# between the two CI providers. Be sure to also change the configuration in +# src/ci/github-actions when changing this file. + +##################################### + # # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. # diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml index 9c5a15a3cf4..485b80398c8 100644 --- a/src/ci/azure-pipelines/master.yml +++ b/src/ci/azure-pipelines/master.yml @@ -1,3 +1,14 @@ +##################################### +## READ BEFORE CHANGING THIS ## +##################################### + +# We're in the process of evaluating GitHub Actions as a possible replacement +# for Azure Pipelines, and at the moment the configuration is duplicated +# between the two CI providers. Be sure to also change the configuration in +# src/ci/github-actions when changing this file. + +##################################### + # # Azure Pipelines job to publish toolstate. Only triggers on pushes to master. # diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml index 1f0be53677d..37c1779b799 100644 --- a/src/ci/azure-pipelines/pr.yml +++ b/src/ci/azure-pipelines/pr.yml @@ -1,3 +1,14 @@ +##################################### +## READ BEFORE CHANGING THIS ## +##################################### + +# We're in the process of evaluating GitHub Actions as a possible replacement +# for Azure Pipelines, and at the moment the configuration is duplicated +# between the two CI providers. Be sure to also change the configuration in +# src/ci/github-actions when changing this file. + +##################################### + # # Azure Pipelines pull request build for Rust # diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index e2f4b62deaf..85ff3e52a84 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -1,3 +1,14 @@ +##################################### +## READ BEFORE CHANGING THIS ## +##################################### + +# We're in the process of evaluating GitHub Actions as a possible replacement +# for Azure Pipelines, and at the moment the configuration is duplicated +# between the two CI providers. Be sure to also change the configuration in +# src/ci/github-actions when changing this file. + +##################################### + # FIXME(linux): need to configure core dumps, enable them, and then dump # backtraces on failure from all core dumps: # diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index 698608795e2..38a0685e0f7 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -1,3 +1,14 @@ +##################################### +## READ BEFORE CHANGING THIS ## +##################################### + +# We're in the process of evaluating GitHub Actions as a possible replacement +# for Azure Pipelines, and at the moment the configuration is duplicated +# between the two CI providers. Be sure to also change the configuration in +# src/ci/github-actions when changing this file. + +##################################### + pr: none trigger: - try