Upload the toolstate to the remote repository.

This commit is contained in:
kennytm 2017-12-04 22:31:57 +08:00
parent 0692b3df2e
commit 519f92f2aa
No known key found for this signature in database
GPG Key ID: FEF6C8051D0E013C
7 changed files with 230 additions and 11 deletions

View File

@ -179,6 +179,17 @@ matrix:
- env: IMAGE=x86_64-gnu-incremental
if: branch = auto
- stage: publish toolstate
if: branch = master AND type = push
before_install: []
install: []
cache: false
sudo: false
script:
MESSAGE_FILE=$(mktemp -t msg.XXXXXX);
. src/ci/docker/x86_64-gnu-tools/repo.sh;
commit_toolstate_change "$MESSAGE_FILE" "$TRAVIS_BUILD_DIR/src/ci/docker/x86_64-gnu-tools/publish.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE"
env:
global:
- SCCACHE_BUCKET=rust-lang-ci-sccache2
@ -186,6 +197,8 @@ env:
- AWS_ACCESS_KEY_ID=AKIAJAMV3QAMMA6AXHFQ
# AWS_SECRET_ACCESS_KEY=...
- secure: "j96XxTVOSUf4s4r4htIxn/fvIa5DWbMgLqWl7r8z2QfgUwscmkMXAwXuFNc7s7bGTpV/+CgDiMFFM6BAFLGKutytIF6oA02s9b+usQYnM0th7YQ2AIgm9GtMTJCJp4AoyfFmh8F2faUICBZlfVLUJ34udHEe35vOklix+0k4WDo="
# TOOLSTATE_REPO_ACCESS_TOKEN=...
- secure: "cFh8thThqEJLC98XKI5pfqflUzOlxsYPRW20AWRaYOOgYHPTiGWypTXiPbGSKaeAXTZoOA+DpQtEmefc0U6lt9dHc7a/MIaK6isFurjlnKYiLOeTruzyu1z7PWCeZ/jKXsU2RK/88DBtlNwfMdaMIeuKj14IVfpepPPL71ETbuk="
before_install:
- zcat $HOME/docker/rust-ci.tar.gz | docker load || true

View File

@ -5,6 +5,8 @@ environment:
AWS_SECRET_ACCESS_KEY:
secure: 7Y+JiquYedOAgnUU26uL0DPzrxmTtR+qIwG6rNKSuWDffqU3vVZxbGXim9QpTO80
SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c
TOOLSTATE_REPO_ACCESS_TOKEN:
secure: PTZiSxJMVUZ0VnMR5i13E4OagbXfglj7pcskDQiKufVrDm13mLoI0vDJAEM35+bY
# By default schannel checks revocation of certificates unlike some other SSL
# backends, but we've historically had problems on CI where a revocation
@ -27,8 +29,8 @@ environment:
# MSVC tools tests
- MSYS_BITS: 64
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py toolstates.json
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=toolstates.json --enable-test-miri
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
# 32/64-bit MinGW builds.
#

View File

@ -99,6 +99,7 @@ exec docker \
--env LOCAL_USER_ID=`id -u` \
--env TRAVIS \
--env TRAVIS_BRANCH \
--env TOOLSTATE_REPO_ACCESS_TOKEN \
--volume "$HOME/.cargo:/cargo" \
--volume "$HOME/rustsrc:$HOME/rustsrc" \
--init \

View File

@ -18,9 +18,10 @@ COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
COPY x86_64-gnu-tools/checktools.sh /tmp/
COPY x86_64-gnu-tools/repo.sh /tmp/
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--enable-test-miri \
--save-toolstates=/tmp/toolstates.json
ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json
ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux

View File

@ -13,7 +13,9 @@
set -eu
X_PY="$1"
TOOLSTATE_FILE="$2"
TOOLSTATE_FILE="$(realpath $2)"
OS="$3"
COMMIT="$(git rev-parse HEAD)"
touch "$TOOLSTATE_FILE"
@ -23,17 +25,22 @@ python2.7 "$X_PY" test --no-fail-fast \
src/tools/rustfmt \
src/tools/miri \
src/tools/clippy
TEST_RESULT=$?
set -e
# FIXME: Upload this file to the repository.
cat "$TOOLSTATE_FILE"
# FIXME: After we can properly inform dev-tool maintainers about failure,
# comment out the `exit 0` below.
if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then
# exit 0
true
. "$(dirname $0)/repo.sh"
MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
echo "($OS CI update)" > "$MESSAGE_FILE"
commit_toolstate_change "$MESSAGE_FILE" \
sed -i "1 a\\
$COMMIT\t$(cat "$TOOLSTATE_FILE")
" "history/$OS.tsv"
rm -f "$MESSAGE_FILE"
exit 0
fi
exit $TEST_RESULT
if grep -q 'Broken\|Compiling' "$TOOLSTATE_FILE"; then
exit 4
fi

View File

@ -0,0 +1,105 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import sys
import re
import json
import copy
import datetime
import collections
# List of people to ping when the status of a tool changed.
MAINTAINERS = {
'miri': '@oli-obk @RalfJung @eddyb',
'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk',
'rls': '@nrc',
'rustfmt': '@nrc',
}
def read_current_status(current_commit, path):
'''Reads build status of `current_commit` from content of `history/*.tsv`
'''
with open(path, 'rU') as f:
for line in f:
(commit, status) = line.split('\t', 1)
if commit == current_commit:
return json.loads(status)
return {}
def update_latest(current_commit, relevant_pr_number, current_datetime):
'''Updates `_data/latest.json` to match build result of the given commit.
'''
with open('_data/latest.json', 'rb+') as f:
latest = json.load(f, object_pairs_hook=collections.OrderedDict)
current_status = {
os: read_current_status(current_commit, 'history/' + os + '.tsv')
for os in ['windows', 'linux']
}
slug = 'rust-lang/rust'
message = '📣 Toolstate changed by {}!\n\nTested on commit {}@{}.\n\n' \
.format(relevant_pr_number, slug, current_commit)
anything_changed = False
for status in latest:
tool = status['tool']
changed = False
for os, s in current_status.items():
old = status[os]
new = s.get(tool, old)
status[os] = new
if new > old:
changed = True
message += '🎉 {} on {}: {}{}.\n' \
.format(tool, os, old, new)
elif new < old:
changed = True
message += '💔 {} on {}: {}{} (cc {}).\n' \
.format(tool, os, old, new, MAINTAINERS[tool])
if changed:
status['commit'] = current_commit
status['datetime'] = current_datetime
anything_changed = True
if not anything_changed:
return ''
f.seek(0)
f.truncate(0)
json.dump(latest, f, indent=4, separators=(',', ': '))
return message
if __name__ == '__main__':
cur_commit = sys.argv[1]
cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
cur_commit_msg = sys.argv[2]
save_message_to_path = sys.argv[3]
relevant_pr_match = re.search('#[0-9]+', cur_commit_msg)
if relevant_pr_match:
relevant_pr_number = 'rust-lang/rust' + relevant_pr_match.group(0)
else:
relevant_pr_number = '<unknown PR>'
message = update_latest(cur_commit, relevant_pr_number, cur_datetime)
if message:
print(message)
with open(save_message_to_path, 'w') as f:
f.write(message)
else:
print('<Nothing changed>')

View File

@ -0,0 +1,90 @@
#!/bin/sh
# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
# This file provides the function `commit_toolstate_change` for pushing a change
# to the `rust-toolstate` repository.
#
# The function relies on a GitHub bot user, which should have a Personal access
# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
# some reason you need to change the token, please update `.travis.yml` and
# `appveyor.yml`:
#
# 1. Generate a new Personal access token:
#
# * Login to the bot account, and go to Settings -> Developer settings ->
# Personal access tokens
# * Click "Generate new token"
# * Enable the "public_repo" permission, then click "Generate token"
# * Copy the generated token (should be a 40-digit hexadecimal number).
# Save it somewhere secure, as the token would be gone once you leave
# the page.
#
# 2. Encrypt the token for Travis CI
#
# * Install the `travis` tool locally (`gem install travis`).
# * Encrypt the token:
# ```
# travis -r rust-lang/rust encrypt \
# TOOLSTATE_REPO_ACCESS_TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
# ```
# * Copy output to replace the existing one in `.travis.yml`.
# * Details of this step can be found in
# <https://docs.travis-ci.com/user/encryption-keys/>
#
# 3. Encrypt the token for AppVeyor
#
# * Login to AppVeyor using your main account, and login as the rust-lang
# organization.
# * Open the ["Encrypt data" tool](https://ci.appveyor.com/tools/encrypt)
# * Paste the 40-digit token into the "Value to encrypt" box, then click
# "Encrypt"
# * Copy the output to replace the existing one in `appveyor.yml`.
# * Details of this step can be found in
# <https://www.appveyor.com/docs/how-to/git-push/>
#
# 4. Replace the email address below if the bot account identity is changed
#
# * See <https://help.github.com/articles/about-commit-email-addresses/>
# if a private email by GitHub is wanted.
commit_toolstate_change() {
OLDFLAGS="$-"
set -eu
git config --global user.email '34210020+rust-toolstate-update@users.noreply.github.com'
git config --global user.name 'Rust Toolstate Update'
git config --global credential.helper store
printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
> "$HOME/.git-credentials"
git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git
cd rust-toolstate
FAILURE=1
MESSAGE_FILE="$1"
shift
for RETRY_COUNT in 1 2 3 4 5; do
"$@"
# `git commit` failing means nothing to commit.
FAILURE=0
git commit -a -F "$MESSAGE_FILE" || break
# On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
FAILURE=1
git fetch origin master
git reset --hard origin/master
done
cd ..
set +eu
set "-$OLDFLAGS"
return $FAILURE
}