OpenE2K/rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ci: add github actions configuration

Browse Source
This commit is contained in:
Pietro Albini 2019-11-26 12:06:30 +01:00
parent 9d5c416037
commit 9beb8f5477
No known key found for this signature in database
GPG Key ID: 3E06ABE80BAAF19C
17 changed files with 1909 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

781
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,781 @@
#############################################################
# WARNING: automatically generated file, DO NOT CHANGE! #
#############################################################
# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
# src/ci/github-actions/ci.yml
#
# Once you make changes to that file you need to run:
#
# ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.
---
name: CI
"on":
push:
branches:
- auto
- try
- master
pull_request:
branches:
- "**"
jobs:
pr:
name: PR
env:
CI_JOB_NAME: "${{ matrix.name }}"
SCCACHE_BUCKET: rust-lang-gha-caches
TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate"
if: "github.event_name == 'pull_request'"
strategy:
matrix:
name:
- mingw-check
- x86_64-gnu-llvm-7
- x86_64-gnu-tools
include:
- name: mingw-check
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-llvm-7
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-tools
env:
CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
os: ubuntu-latest-xl
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:
- name: disable git crlf conversion
run: git config --global core.autocrlf false
shell: bash
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: configure GitHub Actions to kill the build when outdated
uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
with:
github_token: "${{ secrets.github_token }}"
if: success() && !env.SKIP_JOB
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install InnoSetup
run: src/ci/scripts/install-innosetup.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure the build happens on a partition with enough space
run: src/ci/scripts/symlink-build-dir.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2 packages
run: src/ci/scripts/install-msys2-packages.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
shell: "python src/ci/exec-with-shell.py {0}"
try:
name: try
env:
CI_JOB_NAME: "${{ matrix.name }}"
SCCACHE_BUCKET: rust-lang-gha-caches
DEPLOY_BUCKET: rust-lang-gha
TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate"
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues"
TOOLSTATE_PUBLISH: 1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
if: "github.event_name == 'push' && github.ref == 'refs/heads/try'"
strategy:
matrix:
name:
- dist-x86_64-linux
- dist-x86_64-linux-alt
include:
- name: dist-x86_64-linux
os: ubuntu-latest-xl
env: {}
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
os: ubuntu-latest-xl
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:
- name: disable git crlf conversion
run: git config --global core.autocrlf false
shell: bash
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: configure GitHub Actions to kill the build when outdated
uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
with:
github_token: "${{ secrets.github_token }}"
if: success() && !env.SKIP_JOB
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install InnoSetup
run: src/ci/scripts/install-innosetup.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure the build happens on a partition with enough space
run: src/ci/scripts/symlink-build-dir.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2 packages
run: src/ci/scripts/install-msys2-packages.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
shell: "python src/ci/exec-with-shell.py {0}"
auto:
name: auto
env:
CI_JOB_NAME: "${{ matrix.name }}"
SCCACHE_BUCKET: rust-lang-gha-caches
DEPLOY_BUCKET: rust-lang-gha
TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate"
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues"
TOOLSTATE_PUBLISH: 1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
if: "github.event_name == 'push' && github.ref == 'refs/heads/auto'"
strategy:
matrix:
name:
- arm-android
- armhf-gnu
- dist-aarch64-linux
- dist-android
- dist-arm-linux
- dist-armhf-linux
- dist-armv7-linux
- dist-i586-gnu-i586-i686-musl
- dist-i686-freebsd
- dist-i686-linux
- dist-i686-mingw
- dist-i686-msvc
- dist-mips-linux
- dist-mips64-linux
- dist-mips64el-linux
- dist-mipsel-linux
- dist-powerpc-linux
- dist-powerpc64-linux
- dist-powerpc64le-linux
- dist-s390x-linux
- dist-various-1
- dist-various-2
- dist-x86_64-apple
- dist-x86_64-apple-alt
- dist-x86_64-freebsd
- dist-x86_64-linux
- dist-x86_64-linux-alt
- dist-x86_64-mingw
- dist-x86_64-msvc
- dist-x86_64-msvc-alt
- dist-x86_64-musl
- dist-x86_64-netbsd
- i686-gnu
- i686-gnu-nopt
- i686-mingw-1
- i686-mingw-2
- i686-msvc-1
- i686-msvc-2
- mingw-check
- test-various
- wasm32
- x86_64-apple
- x86_64-gnu
- x86_64-gnu-aux
- x86_64-gnu-debug
- x86_64-gnu-distcheck
- x86_64-gnu-full-bootstrap
- x86_64-gnu-llvm-7
- x86_64-gnu-nopt
- x86_64-gnu-tools
- x86_64-mingw-1
- x86_64-mingw-2
- x86_64-msvc-1
- x86_64-msvc-2
- x86_64-msvc-aux
- x86_64-msvc-cargo
- x86_64-msvc-tools
include:
- name: arm-android
os: ubuntu-latest-xl
env: {}
- name: armhf-gnu
os: ubuntu-latest-xl
env: {}
- name: dist-aarch64-linux
os: ubuntu-latest-xl
env: {}
- name: dist-android
os: ubuntu-latest-xl
env: {}
- name: dist-arm-linux
os: ubuntu-latest-xl
env: {}
- name: dist-armhf-linux
os: ubuntu-latest-xl
env: {}
- name: dist-armv7-linux
os: ubuntu-latest-xl
env: {}
- name: dist-i586-gnu-i586-i686-musl
os: ubuntu-latest-xl
env: {}
- name: dist-i686-freebsd
os: ubuntu-latest-xl
env: {}
- name: dist-i686-linux
os: ubuntu-latest-xl
env: {}
- name: dist-mips-linux
os: ubuntu-latest-xl
env: {}
- name: dist-mips64-linux
os: ubuntu-latest-xl
env: {}
- name: dist-mips64el-linux
os: ubuntu-latest-xl
env: {}
- name: dist-mipsel-linux
os: ubuntu-latest-xl
env: {}
- name: dist-powerpc-linux
os: ubuntu-latest-xl
env: {}
- name: dist-powerpc64-linux
os: ubuntu-latest-xl
env: {}
- name: dist-powerpc64le-linux
os: ubuntu-latest-xl
env: {}
- name: dist-s390x-linux
os: ubuntu-latest-xl
env: {}
- name: dist-various-1
os: ubuntu-latest-xl
env: {}
- name: dist-various-2
os: ubuntu-latest-xl
env: {}
- name: dist-x86_64-freebsd
os: ubuntu-latest-xl
env: {}
- name: dist-x86_64-linux
os: ubuntu-latest-xl
env: {}
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
os: ubuntu-latest-xl
- name: dist-x86_64-musl
os: ubuntu-latest-xl
env: {}
- name: dist-x86_64-netbsd
os: ubuntu-latest-xl
env: {}
- name: i686-gnu
os: ubuntu-latest-xl
env: {}
- name: i686-gnu-nopt
os: ubuntu-latest-xl
env: {}
- name: mingw-check
os: ubuntu-latest-xl
env: {}
- name: test-various
os: ubuntu-latest-xl
env: {}
- name: wasm32
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-aux
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-debug
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-distcheck
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-full-bootstrap
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-llvm-7
env:
RUST_BACKTRACE: 1
os: ubuntu-latest-xl
- name: x86_64-gnu-nopt
os: ubuntu-latest-xl
env: {}
- name: x86_64-gnu-tools
env:
DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
os: ubuntu-latest-xl
- name: dist-x86_64-apple
env:
SCRIPT: "./x.py dist"
RUST_CONFIGURE_ARGS: "--target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
DIST_REQUIRE_ALL_TOOLS: 1
os: macos-latest
- name: dist-x86_64-apple-alt
env:
SCRIPT: "./x.py dist"
RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
os: macos-latest
- name: x86_64-apple
env:
SCRIPT: "./x.py test"
RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc"
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.8
MACOSX_STD_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
os: macos-latest
- name: x86_64-msvc-1
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"
SCRIPT: make ci-subset-1
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: x86_64-msvc-2
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"
SCRIPT: make ci-subset-2
os: windows-latest-xl
- name: i686-msvc-1
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc"
SCRIPT: make ci-subset-1
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: i686-msvc-2
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc"
SCRIPT: make ci-subset-2
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: x86_64-msvc-aux
env:
RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc"
os: windows-latest-xl
- name: x86_64-msvc-cargo
env:
SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc"
VCVARS_BAT: vcvars64.bat
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: x86_64-msvc-tools
env:
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json"
os: windows-latest-xl
- name: i686-mingw-1
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu"
SCRIPT: make ci-mingw-subset-1
CUSTOM_MINGW: 1
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: i686-mingw-2
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu"
SCRIPT: make ci-mingw-subset-2
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: x86_64-mingw-1
env:
SCRIPT: make ci-mingw-subset-1
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu"
CUSTOM_MINGW: 1
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
os: windows-latest-xl
- name: x86_64-mingw-2
env:
SCRIPT: make ci-mingw-subset-2
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu"
CUSTOM_MINGW: 1
os: windows-latest-xl
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-i686-msvc
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --target=i586-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-i686-mingw
env:
RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler"
SCRIPT: python x.py dist
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-x86_64-mingw
env:
SCRIPT: python x.py dist
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler"
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-latest-xl
- name: dist-x86_64-msvc-alt
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
SCRIPT: python x.py dist
os: windows-latest-xl
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
steps:
- name: disable git crlf conversion
run: git config --global core.autocrlf false
shell: bash
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: configure GitHub Actions to kill the build when outdated
uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
with:
github_token: "${{ secrets.github_token }}"
if: success() && !env.SKIP_JOB
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install awscli
run: src/ci/scripts/install-awscli.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install sccache
run: src/ci/scripts/install-sccache.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install clang
run: src/ci/scripts/install-clang.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install WIX
run: src/ci/scripts/install-wix.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install InnoSetup
run: src/ci/scripts/install-innosetup.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure the build happens on a partition with enough space
run: src/ci/scripts/symlink-build-dir.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MSYS2 packages
run: src/ci/scripts/install-msys2-packages.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: install ninja
run: src/ci/scripts/install-ninja.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
shell: "python src/ci/exec-with-shell.py {0}"
master:
name: master
runs-on: ubuntu-latest
env:
SCCACHE_BUCKET: rust-lang-gha-caches
DEPLOY_BUCKET: rust-lang-gha
TOOLSTATE_REPO: "https://github.com/pietroalbini/rust-toolstate"
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/pietroalbini/rust-toolstate/issues"
TOOLSTATE_PUBLISH: 1
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
if: "github.event_name == 'push' && github.ref == 'refs/heads/master'"
steps:
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: publish toolstate
run: src/ci/publish_toolstate.sh
env:
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
shell: "python src/ci/exec-with-shell.py {0}"
if: success() && !env.SKIP_JOB
try-success:
needs:
- try
if: success()
steps:
- name: mark the job as a success
run: exit 0
name: bors build finished
runs-on: ubuntu-latest
try-failure:
needs:
- try
if: "!success()"
steps:
- name: mark the job as a failure
run: exit 1
name: bors build finished
runs-on: ubuntu-latest
auto-success:
needs:
- auto
if: success()
steps:
- name: mark the job as a success
run: exit 0
name: bors build finished
runs-on: ubuntu-latest
auto-failure:
needs:
- auto
if: "!success()"
steps:
- name: mark the job as a failure
run: exit 1
name: bors build finished
runs-on: ubuntu-latest

56
Cargo.lock
View File

@ -443,7 +443,7 @@ dependencies = [
"textwrap",
"unicode-width",
"vec_map",
"yaml-rust",
"yaml-rust 0.3.5",
]
[[package]]
@ -1026,6 +1026,14 @@ dependencies = [
"walkdir",
]
[[package]]
name = "expand-yaml-anchors"
version = "0.1.0"
dependencies = [
"yaml-merge-keys",
"yaml-rust 0.4.3",
]
[[package]]
name = "failure"
version = "0.1.5"
@ -1830,6 +1838,12 @@ dependencies = [
name = "linkchecker"
version = "0.1.0"
[[package]]
name = "linked-hash-map"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
[[package]]
name = "lock_api"
version = "0.3.1"
@ -4854,6 +4868,26 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9fb62ff737e573b1e677459bea6fd023cd5d6e868c3242d3cdf3ef2f0554824"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24069c0ba08aab54289d6a25f5036d94afc61e1538bbc42ae5501df141c9027d"
dependencies = [
"proc-macro2 1.0.3",
"quote 1.0.2",
"syn 1.0.11",
]
[[package]]
name = "thread_local"
version = "0.3.6"
@ -5536,8 +5570,28 @@ dependencies = [
"lzma-sys",
]
[[package]]
name = "yaml-merge-keys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59893318ba3ad2b704498c7761214a10eaf42c5f838bce9fc0145bf2ba658cfa"
dependencies = [
"lazy_static 1.4.0",
"thiserror",
"yaml-rust 0.4.3",
]
[[package]]
name = "yaml-rust"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"
[[package]]
name = "yaml-rust"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"
dependencies = [
"linked-hash-map",
]

1
Cargo.toml
View File

@ -24,6 +24,7 @@ members = [
"src/tools/miri",
"src/tools/rustdoc-themes",
"src/tools/unicode-table-generator",
"src/tools/expand-yaml-anchors",
]
exclude = [
"build",

5
src/bootstrap/builder.rs
View File

@ -21,6 +21,7 @@ use crate::doc;
use crate::flags::Subcommand;
use crate::install;
use crate::native;
use crate::run;
use crate::test;
use crate::tool;
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir};
@ -313,6 +314,7 @@ pub enum Kind {
Dist,
Doc,
Install,
Run,
}
impl<'a> Builder<'a> {
@ -353,6 +355,7 @@ impl<'a> Builder<'a> {
}
Kind::Test => describe!(
crate::toolstate::ToolStateCheck,
test::ExpandYamlAnchors,
test::Tidy,
test::Ui,
test::CompileFail,
@ -454,6 +457,7 @@ impl<'a> Builder<'a> {
install::Src,
install::Rustc
),
Kind::Run => describe!(run::ExpandYamlAnchors,),
}
}
@ -507,6 +511,7 @@ impl<'a> Builder<'a> {
Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(),
};

24
src/bootstrap/flags.rs
View File

@ -86,6 +86,9 @@ pub enum Subcommand {
Install {
paths: Vec<PathBuf>,
},
Run {
paths: Vec<PathBuf>,
},
}
impl Default for Subcommand {
@ -113,6 +116,7 @@ Subcommands:
clean Clean out build directories
dist Build distribution artifacts
install Install distribution artifacts
run Run tools contained in this repository
To learn more about a subcommand, run `./x.py <subcommand> -h`",
);
@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
|| (s == "clean")
|| (s == "dist")
|| (s == "install")
|| (s == "run")
});
let subcommand = match subcommand {
Some(s) => s,
@ -400,6 +405,18 @@ Arguments:
./x.py doc --stage 1",
);
}
"run" => {
subcommand_help.push_str(
"\n
Arguments:
This subcommand accepts a number of paths to tools to build and run. For
example:
./x.py run src/tool/expand-yaml-anchors
At least a tool needs to be called.",
);
}
_ => {}
};
// Get any optional paths which occur after the subcommand
@ -468,6 +485,13 @@ Arguments:
"fmt" => Subcommand::Format { check: matches.opt_present("check") },
"dist" => Subcommand::Dist { paths },
"install" => Subcommand::Install { paths },
"run" => {
if paths.is_empty() {
println!("\nrun requires at least a path!\n");
usage(1, &opts, &subcommand_help, &extra_help);
}
Subcommand::Run { paths }
}
_ => {
usage(1, &opts, &subcommand_help, &extra_help);
}

1
src/bootstrap/lib.rs
View File

@ -140,6 +140,7 @@ mod format;
mod install;
mod metadata;
mod native;
mod run;
mod sanity;
mod test;
mod tool;

45
src/bootstrap/run.rs Normal file
View File

@ -0,0 +1,45 @@
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::tool::Tool;
use std::process::Command;
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct ExpandYamlAnchors;
impl Step for ExpandYamlAnchors {
type Output = ();
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
/// Runs the `expand-yaml_anchors` tool.
///
/// This tool in `src/tools` read the CI configuration files written in YAML and expands the
/// anchors in them, since GitHub Actions doesn't support them.
fn run(self, builder: &Builder<'_>) {
builder.info("Expanding YAML anchors in the GitHub Actions configuration");
try_run(
builder,
&mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
);
}
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/expand-yaml-anchors")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(ExpandYamlAnchors);
}
}
fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
if !builder.fail_fast {
if !builder.try_run(cmd) {
let mut failures = builder.delayed_failures.borrow_mut();
failures.push(format!("{:?}", cmd));
return false;
}
} else {
builder.run(cmd);
}
true
}

29
src/bootstrap/test.rs
View File

@ -750,6 +750,35 @@ impl Step for Tidy {
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct ExpandYamlAnchors;
impl Step for ExpandYamlAnchors {
type Output = ();
const ONLY_HOSTS: bool = true;
/// Ensure the `generate-ci-config` tool was run locally.
///
/// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
/// appropriate configuration for all our CI providers. This step ensures the tool was called
/// by the user before committing CI changes.
fn run(self, builder: &Builder<'_>) {
builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
try_run(
builder,
&mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
);
}
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/expand-yaml-anchors")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(ExpandYamlAnchors);
}
}
fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
builder.out.join(host).join("test")
}

1
src/bootstrap/tool.rs
View File

@ -378,6 +378,7 @@ bootstrap_tool!(
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
);
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

3
src/ci/docker/mingw-check/Dockerfile
View File

@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh
COPY mingw-check/validate-toolstate.sh /scripts/
ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \
python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
python2.7 ../x.py build --stage 0 src/tools/build-manifest && \
python2.7 ../x.py test --stage 0 src/tools/compiletest && \
python2.7 ../x.py test src/tools/tidy && \

16
src/ci/exec-with-shell.py Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
# A simple wrapper that forwards the arguments to bash, unless the
# CI_OVERRIDE_SHELL environment variable is present: in that case the content
# of that environment variable is used as the shell path.
import os
import sys
import subprocess
try:
shell = os.environ["CI_OVERRIDE_SHELL"]
except KeyError:
shell = "bash"
res = subprocess.call([shell] + sys.argv[1:])
sys.exit(res)

707
src/ci/github-actions/ci.yml Normal file
View File

@ -0,0 +1,707 @@
######################################################
# WARNING! Action needed when changing this file #
######################################################
# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
# CI configuration stored on the repository. To work around that this file is
# expanded by a tool in the repository, and the expansion is committed as well.
#
# After you make any change to the file you'll need to run this command:
#
# ./x.py run src/tools/expand-yaml-anchors
#
# ...and commit the file it updated in addition to this one. If you forget this
# step CI will fail.
---
###############################
# YAML Anchors Definition #
###############################
# This key contains most of the YAML anchors that will be used later in the
# document. YAML anchors allows us to greatly reduce duplication inside the CI
# configuration by reusing parts of the configuration.
#
# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
# content in another place with `*anchor-name`. The special `<<` map key merges
# the content of the map with the content of the anchor (or list of anchors).
#
# The expand-yaml-anchors tool will automatically remove this block from the
# output YAML file.
x--expand-yaml-anchors--remove:
- &shared-ci-variables
CI_JOB_NAME: ${{ matrix.name }}
- &public-variables
SCCACHE_BUCKET: rust-lang-gha-caches
TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
- &prod-variables
SCCACHE_BUCKET: rust-lang-gha-caches
DEPLOY_BUCKET: rust-lang-gha
TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
TOOLSTATE_PUBLISH: 1
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
# rotate them in a single branch while keeping the old key in another
# branch, which wouldn't be possible if the key was named with the kind
# (caches, artifacts...).
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
- &base-job
env: {}
- &job-linux-xl
os: ubuntu-latest-xl
<<: *base-job
- &job-macos-xl
os: macos-latest # We don't have an XL builder for this
<<: *base-job
- &job-windows-xl
os: windows-latest-xl
<<: *base-job
- &step
if: success() && !env.SKIP_JOB
- &step-run
<<: *step
# While on Linux and macOS builders it just forwards the arguments to the
# system bash, this wrapper allows switching from the host's bash.exe to
# the one we install along with MSYS2 mid-build on Windows.
#
# Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
# environment variable is set pointing to our MSYS2's bash.exe. From that
# moment the host's bash.exe will not be called anymore.
#
# This is needed because we can't launch our own bash.exe from the host
# bash.exe, as that would load two different cygwin1.dll in memory, causing
# "cygwin heap mismatch" errors.
shell: python src/ci/exec-with-shell.py {0}
- &base-ci-job
timeout-minutes: 600
runs-on: "${{ matrix.os }}"
env: *shared-ci-variables
steps:
- name: disable git crlf conversion
run: git config --global core.autocrlf false
shell: bash
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: configure GitHub Actions to kill the build when outdated
uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
with:
github_token: "${{ secrets.github_token }}"
<<: *step
- name: add extra environment variables
run: src/ci/scripts/setup-environment.sh
env:
# Since it's not possible to merge `${{ matrix.env }}` with the other
# variables in `job.<name>.env`, the variables defined in the matrix
# are passed to the `setup-environment.sh` script encoded in JSON,
# which then uses log commands to actually set them.
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
<<: *step-run
- name: decide whether to skip this job
run: src/ci/scripts/should-skip-this.sh
<<: *step-run
- name: collect CPU statistics
run: src/ci/scripts/collect-cpu-stats.sh
<<: *step-run
- name: show the current environment
run: src/ci/scripts/dump-environment.sh
<<: *step-run
- name: install awscli
run: src/ci/scripts/install-awscli.sh
<<: *step-run
- name: install sccache
run: src/ci/scripts/install-sccache.sh
<<: *step-run
- name: install clang
run: src/ci/scripts/install-clang.sh
<<: *step-run
- name: install WIX
run: src/ci/scripts/install-wix.sh
<<: *step-run
- name: install InnoSetup
run: src/ci/scripts/install-innosetup.sh
<<: *step-run
- name: ensure the build happens on a partition with enough space
run: src/ci/scripts/symlink-build-dir.sh
<<: *step-run
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
<<: *step-run
- name: install MSYS2
run: src/ci/scripts/install-msys2.sh
<<: *step-run
- name: install MSYS2 packages
run: src/ci/scripts/install-msys2-packages.sh
<<: *step-run
- name: install MinGW
run: src/ci/scripts/install-mingw.sh
<<: *step-run
- name: install ninja
run: src/ci/scripts/install-ninja.sh
<<: *step-run
- name: enable ipv6 on Docker
run: src/ci/scripts/enable-docker-ipv6.sh
<<: *step-run
# Disable automatic line ending conversion (again). On Windows, when we're
# installing dependencies, something switches the git configuration directory or
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
# be multiple -- but this should ensure submodules are checked out with the
# appropriate line endings.
- name: disable git crlf conversion
run: src/ci/scripts/disable-git-crlf-conversion.sh
<<: *step-run
- name: checkout submodules
run: src/ci/scripts/checkout-submodules.sh
<<: *step-run
- name: ensure line endings are correct
run: src/ci/scripts/verify-line-endings.sh
<<: *step-run
- name: run the build
run: src/ci/scripts/run-build-from-ci.sh
env:
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
<<: *step-run
- name: upload artifacts to S3
run: src/ci/scripts/upload-artifacts.sh
env:
AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
# builders *should* have the AWS credentials available. Still, explicitly
# adding the condition is helpful as this way CI will not silently skip
# deploying artifacts from a dist builder if the variables are misconfigured,
# erroring about invalid credentials instead.
if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
<<: *step-run
# These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
# Check out their documentation for more information on why they're needed.
- &base-outcome-job
name: bors build finished
runs-on: ubuntu-latest
- &base-success-job
if: success()
steps:
- name: mark the job as a success
run: exit 0
<<: *base-outcome-job
- &base-failure-job
if: "!success()"
steps:
- name: mark the job as a failure
run: exit 1
<<: *base-outcome-job
###########################
# Builders definition #
###########################
name: CI
on:
push:
branches:
- auto
- try
- master
pull_request:
branches:
- "**"
jobs:
pr:
<<: *base-ci-job
name: PR
env:
<<: [*shared-ci-variables, *public-variables]
if: github.event_name == 'pull_request'
strategy:
matrix:
name:
- mingw-check
- x86_64-gnu-llvm-7
- x86_64-gnu-tools
include:
- name: mingw-check
<<: *job-linux-xl
- name: x86_64-gnu-llvm-7
<<: *job-linux-xl
- name: x86_64-gnu-tools
env:
CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
<<: *job-linux-xl
try:
<<: *base-ci-job
name: try
env:
<<: [*shared-ci-variables, *prod-variables]
if: github.event_name == 'push' && github.ref == 'refs/heads/try'
strategy:
matrix:
name:
- dist-x86_64-linux
- dist-x86_64-linux-alt
include:
- name: dist-x86_64-linux
<<: *job-linux-xl
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
<<: *job-linux-xl
auto:
<<: *base-ci-job
name: auto
env:
<<: [*shared-ci-variables, *prod-variables]
if: github.event_name == 'push' && github.ref == 'refs/heads/auto'
strategy:
matrix:
name:
- arm-android
- armhf-gnu
- dist-aarch64-linux
- dist-android
- dist-arm-linux
- dist-armhf-linux
- dist-armv7-linux
- dist-i586-gnu-i586-i686-musl
- dist-i686-freebsd
- dist-i686-linux
- dist-i686-mingw
- dist-i686-msvc
- dist-mips-linux
- dist-mips64-linux
- dist-mips64el-linux
- dist-mipsel-linux
- dist-powerpc-linux
- dist-powerpc64-linux
- dist-powerpc64le-linux
- dist-s390x-linux
- dist-various-1
- dist-various-2
- dist-x86_64-apple
- dist-x86_64-apple-alt
- dist-x86_64-freebsd
- dist-x86_64-linux
- dist-x86_64-linux-alt
- dist-x86_64-mingw
- dist-x86_64-msvc
- dist-x86_64-msvc-alt
- dist-x86_64-musl
- dist-x86_64-netbsd
- i686-gnu
- i686-gnu-nopt
- i686-mingw-1
- i686-mingw-2
- i686-msvc-1
- i686-msvc-2
- mingw-check
- test-various
- wasm32
- x86_64-apple
- x86_64-gnu
- x86_64-gnu-aux
- x86_64-gnu-debug
- x86_64-gnu-distcheck
- x86_64-gnu-full-bootstrap
- x86_64-gnu-llvm-7
- x86_64-gnu-nopt
- x86_64-gnu-tools
- x86_64-mingw-1
- x86_64-mingw-2
- x86_64-msvc-1
- x86_64-msvc-2
- x86_64-msvc-aux
- x86_64-msvc-cargo
- x86_64-msvc-tools
include:
#############################
# Linux/Docker builders #
#############################
- name: arm-android
<<: *job-linux-xl
- name: armhf-gnu
<<: *job-linux-xl
- name: dist-aarch64-linux
<<: *job-linux-xl
- name: dist-android
<<: *job-linux-xl
- name: dist-arm-linux
<<: *job-linux-xl
- name: dist-armhf-linux
<<: *job-linux-xl
- name: dist-armv7-linux
<<: *job-linux-xl
- name: dist-i586-gnu-i586-i686-musl
<<: *job-linux-xl
- name: dist-i686-freebsd
<<: *job-linux-xl
- name: dist-i686-linux
<<: *job-linux-xl
- name: dist-mips-linux
<<: *job-linux-xl
- name: dist-mips64-linux
<<: *job-linux-xl
- name: dist-mips64el-linux
<<: *job-linux-xl
- name: dist-mipsel-linux
<<: *job-linux-xl
- name: dist-powerpc-linux
<<: *job-linux-xl
- name: dist-powerpc64-linux
<<: *job-linux-xl
- name: dist-powerpc64le-linux
<<: *job-linux-xl
- name: dist-s390x-linux
<<: *job-linux-xl
- name: dist-various-1
<<: *job-linux-xl
- name: dist-various-2
<<: *job-linux-xl
- name: dist-x86_64-freebsd
<<: *job-linux-xl
- name: dist-x86_64-linux
<<: *job-linux-xl
- name: dist-x86_64-linux-alt
env:
IMAGE: dist-x86_64-linux
<<: *job-linux-xl
- name: dist-x86_64-musl
<<: *job-linux-xl
- name: dist-x86_64-netbsd
<<: *job-linux-xl
- name: i686-gnu
<<: *job-linux-xl
- name: i686-gnu-nopt
<<: *job-linux-xl
- name: mingw-check
<<: *job-linux-xl
- name: test-various
<<: *job-linux-xl
- name: wasm32
<<: *job-linux-xl
- name: x86_64-gnu
<<: *job-linux-xl
- name: x86_64-gnu-aux
<<: *job-linux-xl
- name: x86_64-gnu-debug
<<: *job-linux-xl
- name: x86_64-gnu-distcheck
<<: *job-linux-xl
- name: x86_64-gnu-full-bootstrap
<<: *job-linux-xl
- name: x86_64-gnu-llvm-7
env:
RUST_BACKTRACE: 1
<<: *job-linux-xl
- name: x86_64-gnu-nopt
<<: *job-linux-xl
- name: x86_64-gnu-tools
env:
DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
<<: *job-linux-xl
####################
# macOS Builders #
####################
- name: dist-x86_64-apple
env:
SCRIPT: ./x.py dist
RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-macos-xl
- name: dist-x86_64-apple-alt
env:
SCRIPT: ./x.py dist
RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
<<: *job-macos-xl
- name: x86_64-apple
env:
SCRIPT: ./x.py test
RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.8
MACOSX_STD_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
<<: *job-macos-xl
######################
# Windows Builders #
######################
- name: x86_64-msvc-1
env:
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
SCRIPT: make ci-subset-1
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: x86_64-msvc-2
env:
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
SCRIPT: make ci-subset-2
<<: *job-windows-xl
- name: i686-msvc-1
env:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
SCRIPT: make ci-subset-1
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: i686-msvc-2
env:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
SCRIPT: make ci-subset-2
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: x86_64-msvc-aux
env:
RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
<<: *job-windows-xl
- name: x86_64-msvc-cargo
env:
SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
VCVARS_BAT: vcvars64.bat
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: x86_64-msvc-tools
env:
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
<<: *job-windows-xl
# 32/64-bit MinGW builds.
#
# We are using MinGW with posix threads since LLVM does not compile with
# the win32 threads version due to missing support for C++'s std::thread.
#
# Instead of relying on the MinGW version installed on appveryor we download
# and install one ourselves so we won't be surprised by changes to appveyor's
# build image.
#
# Finally, note that the downloads below are all in the `rust-lang-ci` S3
# bucket, but they cleraly didn't originate there! The downloads originally
# came from the mingw-w64 SourceForge download site. Unfortunately
# SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
- name: i686-mingw-1
env:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
SCRIPT: make ci-mingw-subset-1
CUSTOM_MINGW: 1
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: i686-mingw-2
env:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
SCRIPT: make ci-mingw-subset-2
CUSTOM_MINGW: 1
<<: *job-windows-xl
- name: x86_64-mingw-1
env:
SCRIPT: make ci-mingw-subset-1
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
CUSTOM_MINGW: 1
# FIXME(#59637)
NO_DEBUG_ASSERTIONS: 1
NO_LLVM_ASSERTIONS: 1
<<: *job-windows-xl
- name: x86_64-mingw-2
env:
SCRIPT: make ci-mingw-subset-2
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
CUSTOM_MINGW: 1
<<: *job-windows-xl
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: >-
--build=x86_64-pc-windows-msvc
--target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
- name: dist-i686-msvc
env:
RUST_CONFIGURE_ARGS: >-
--build=i686-pc-windows-msvc
--target=i586-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
- name: dist-i686-mingw
env:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
SCRIPT: python x.py dist
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
- name: dist-x86_64-mingw
env:
SCRIPT: python x.py dist
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-xl
- name: dist-x86_64-msvc-alt
env:
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
SCRIPT: python x.py dist
<<: *job-windows-xl
master:
name: master
runs-on: ubuntu-latest
env:
<<: [*prod-variables]
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
steps:
- name: checkout the source code
uses: actions/checkout@v1
with:
fetch-depth: 2
- name: publish toolstate
run: src/ci/publish_toolstate.sh
env:
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
<<: *step-run
# These jobs don't actually test anything, but they're used to tell bors the
# build completed, as there is no practical way to detect when a workflow is
# successful listening to webhooks only.
try-success:
needs: [try]
<<: *base-success-job
try-failure:
needs: [try]
<<: *base-failure-job
auto-success:
needs: [auto]
<<: *base-success-job
auto-failure:
needs: [auto]
<<: *base-failure-job

4
src/ci/scripts/install-mingw.sh
View File

@ -50,8 +50,8 @@ if isWindows; then
esac
if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
mingw-w64-$arch-gcc mingw-w64-$arch-python2
pacman -S --noconfirm --needed mingw-w64-$arch-toolchain \
mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2
ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
else
mingw_dir="mingw${bits}"

3
src/ci/scripts/install-msys2.sh
View File

@ -22,4 +22,7 @@ if isWindows; then
rm msys2.nupkg chocolatey-core.extension.nupkg
mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}"
ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin"
echo "switching shell to use our own bash"
ciCommandSetEnv CI_OVERRIDE_SHELL "$(ciCheckoutPath)/msys2/usr/bin/bash.exe"
fi

34
src/ci/scripts/setup-environment.sh
View File

@ -11,16 +11,34 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
# Since matrix variables are readonly in Azure Pipelines, we take
# INITIAL_RUST_CONFIGURE_ARGS and establish RUST_CONFIGURE_ARGS
# which downstream steps can alter
# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
# which was introduced in Bash 4.2
if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
INITIAL_RUST_CONFIG=""
echo "No initial Rust configure args set"
else
INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
if isAzurePipelines; then
# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
# which was introduced in Bash 4.2
if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
INITIAL_RUST_CONFIG=""
echo "No initial Rust configure args set"
else
INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
fi
fi
# Load extra environment variables
vars="${EXTRA_VARIABLES-}"
echo "${vars}" | jq '' >/dev/null # Validate JSON and exit on errors
for key in $(echo "${vars}" | jq "keys[]" -r); do
# On Windows, for whatever reason, $key contains the BOM character in it,
# and that messes up `jq ".${key}"`. This line strips the BOM from the key.
#
# https://unix.stackexchange.com/a/381263
key="$(echo "${key}" | sed '1s/^\xEF\xBB\xBF//')"
echo "adding extra environment variable ${key}"
value="$(echo "${vars}" | jq ".${key}" -r)"
export "${key}"="${value}"
ciCommandSetEnv "${key}" "${value}"
done
# Builders starting with `dist-` are dist builders, but if they also end with
# `-alt` they are alternate dist builders.
if [[ "${CI_JOB_NAME}" = dist-* ]]; then

9
src/tools/expand-yaml-anchors/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "expand-yaml-anchors"
version = "0.1.0"
authors = ["Pietro Albini <pietro@pietroalbini.org>"]
edition = "2018"
[dependencies]
yaml-rust = "0.4.3"
yaml-merge-keys = "0.4.0"

202
src/tools/expand-yaml-anchors/src/main.rs Normal file
View File

@ -0,0 +1,202 @@
use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
/// List of directories containing files to expand. The first tuple element is the source
/// directory, while the second tuple element is the destination directory.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
("src/ci/github-actions", ".github/workflows"),
];
/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";
/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
# WARNING: automatically generated file, DO NOT CHANGE! #
#############################################################
# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
# {source}
#
# Once you make changes to that file you need to run:
#
# ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.
";
enum Mode {
Check,
Generate,
}
struct App {
mode: Mode,
base: PathBuf,
}
impl App {
fn from_args() -> Result<Self, Box<dyn Error>> {
// Parse CLI arguments
let args = std::env::args().skip(1).collect::<Vec<_>>();
let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
&["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
&["check", ref base] => (Mode::Check, PathBuf::from(base)),
_ => {
eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
std::process::exit(1);
}
};
Ok(App { mode, base })
}
fn run(&self) -> Result<(), Box<dyn Error>> {
for (source, dest) in TO_EXPAND {
let source = self.base.join(source);
let dest = self.base.join(dest);
for entry in std::fs::read_dir(&source)? {
let path = entry?.path();
if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
continue;
}
let dest_path = dest.join(path.file_name().unwrap());
self.expand(&path, &dest_path).with_context(|| match self.mode {
Mode::Generate => format!(
"failed to expand {} into {}",
self.path(&path),
self.path(&dest_path)
),
Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
})?;
}
}
Ok(())
}
fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
let content = std::fs::read_to_string(source)
.with_context(|| format!("failed to read {}", self.path(source)))?;
let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string());
let documents = YamlLoader::load_from_str(&content)
.with_context(|| format!("failed to parse {}", self.path(source)))?;
for mut document in documents.into_iter() {
document = yaml_merge_keys::merge_keys(document)
.with_context(|| format!("failed to expand {}", self.path(source)))?;
document = filter_document(document);
YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
context: "failed to serialize the expanded yaml".into(),
source: Box::new(err),
})?;
buf.push('\n');
}
match self.mode {
Mode::Check => {
let old = std::fs::read_to_string(dest)
.with_context(|| format!("failed to read {}", self.path(dest)))?;
if old != buf {
return Err(Box::new(StrError(format!(
"{} and {} are different",
self.path(source),
self.path(dest),
))));
}
}
Mode::Generate => {
std::fs::write(dest, buf.as_bytes())
.with_context(|| format!("failed to write to {}", self.path(dest)))?;
}
}
Ok(())
}
fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
path.strip_prefix(&self.base).unwrap_or(path).display()
}
}
fn filter_document(document: Yaml) -> Yaml {
match document {
Yaml::Hash(map) => Yaml::Hash(
map.into_iter()
.filter(|(key, _)| {
if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
})
.map(|(key, value)| (filter_document(key), filter_document(value)))
.collect(),
),
Yaml::Array(vec) => {
Yaml::Array(vec.into_iter().map(|item| filter_document(item)).collect())
}
other => other,
}
}
fn main() {
if let Err(err) = App::from_args().and_then(|app| app.run()) {
eprintln!("error: {}", err);
let mut source = err.as_ref() as &dyn Error;
while let Some(err) = source.source() {
eprintln!("caused by: {}", err);
source = err;
}
std::process::exit(1);
}
}
#[derive(Debug)]
struct StrError(String);
impl Error for StrError {}
impl std::fmt::Display for StrError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
std::fmt::Display::fmt(&self.0, f)
}
}
#[derive(Debug)]
struct WithContext {
context: String,
source: Box<dyn Error>,
}
impl std::fmt::Display for WithContext {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.context)
}
}
impl Error for WithContext {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(self.source.as_ref())
}
}
pub(crate) trait ResultExt<T> {
fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}
impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
match self {
Ok(ok) => Ok(ok),
Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
}
}
}