Auto merge of #4675 - lzutao:improve-shellscript, r=phansch

build: improve script and travis config * fix a diff failure on windows See https://travis-ci.com/rust-lang/rust-clippy/jobs/245971932#L1625 for an example. * use cmp instead of diff > /dev/null * clone single branch instead of clone then checking out * do not decrypt key if have no diff change changelog: none
2019-10-24 05:20:45 +00:00 · 2019-10-24 05:20:45 +00:00 · 87536f00e3
commit 87536f00e3
parent 850dfdae60 088d528be9
10 changed files with 106 additions and 112 deletions
--- a/.github/deploy.sh
+++ b/.github/deploy.sh
@ -13,11 +13,7 @@ SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:}
 SHA=$(git rev-parse --verify HEAD)

 # Clone the existing gh-pages for this repo into out/
-(
-    git clone "$REPO" out
-    cd out
-    git checkout $TARGET_BRANCH
-)
+git clone --quiet --single-branch --branch "$TARGET_BRANCH" "$REPO" out

 echo "Removing the current docs for master"
 rm -rf out/master/ || exit 0
@ -27,7 +23,7 @@ mkdir out/master/
 cp util/gh-pages/index.html out/master
 python ./util/export.py out/master/lints.json

-if [ -n "$TRAVIS_TAG" ]; then
+if [[ -n "$TRAVIS_TAG" ]]; then
    echo "Save the doc for the current tag ($TRAVIS_TAG) and point current/ to it"
    cp -r out/master "out/$TRAVIS_TAG"
    rm -f out/current
@ -35,27 +31,36 @@ if [ -n "$TRAVIS_TAG" ]; then
 fi

 # Generate version index that is shown as root index page
-(
-    cp util/gh-pages/versions.html out/index.html
+cp util/gh-pages/versions.html out/index.html
+pushd out

-    cd out
-    python -c '\
-        import os, json;\
-        print json.dumps([\
-            dir for dir in os.listdir(".")\
-            if not dir.startswith(".") and os.path.isdir(dir)\
-        ])' > versions.json
-)
+cat <<-EOF | python - > versions.json
+import os, json
+print json.dumps([
+    dir for dir in os.listdir(".") if not dir.startswith(".") and os.path.isdir(dir)
+])
+EOF
+popd

 # Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
+if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] || [[ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]]; then
    # Tags should deploy
-    if [ -z "$TRAVIS_TAG" ]; then
+    if [[ -z "$TRAVIS_TAG" ]]; then
        echo "Generated, won't push"
        exit 0
    fi
 fi

+# Now let's go have some fun with the cloned repo
+cd out
+git config user.name "Travis CI"
+git config user.email "travis@ci.invalid"
+
+if git diff --exit-code --quiet; then
+    echo "No changes to the output on this push; exiting."
+    exit 0
+fi
+
 # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc
 ENCRYPTION_LABEL=e3a2d77100be
 ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
@ -64,19 +69,9 @@ ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
 ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
 openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" -in .github/deploy_key.enc -out .github/deploy_key -d
 chmod 600 .github/deploy_key
-eval $(ssh-agent -s)
+eval "$(ssh-agent -s)"
 ssh-add .github/deploy_key

-# Now let's go have some fun with the cloned repo
-cd out
-git config user.name "Travis CI"
-git config user.email "travis@ci.invalid"
-
-if [ -z "$(git diff --exit-code)" ]; then
-    echo "No changes to the output on this push; exiting."
-    exit 0
-fi
-
 git add .
 git commit -m "Automatic deploy to GitHub Pages: ${SHA}"

--- a/.travis.yml
+++ b/.travis.yml
@ -1,11 +1,6 @@
 dist: xenial
 language: bash

-os:
-  - linux
-  - osx
-  - windows
-
 branches:
  # Don't build these branches
  except:
@ -34,13 +29,12 @@ install:
      if ! rustup component add rustfmt; then
        cargo install -Z install-upgrade --git https://github.com/rust-lang/rustfmt --bin rustfmt
      fi
-      if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+      if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
        . $HOME/.nvm/nvm.sh
        nvm install stable
        nvm use stable
        npm install remark-cli remark-lint
-      fi
-      if [ "$TRAVIS_OS_NAME" == "windows" ]; then
+      elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
        choco install windows-sdk-10.1
      fi
    fi
@ -53,11 +47,9 @@ matrix:
  include:
    # Builds that are executed for every PR
    - os: osx # run base tests on both platforms
-      env: BASE_TESTS=true
    - os: linux
-      env: BASE_TESTS=true
    - os: windows
-      env: CARGO_INCREMENTAL=0 BASE_TESTS=true OS_WINDOWS=true
+      env: CARGO_INCREMENTAL=0 OS_WINDOWS=true

    # Builds that are only executed when a PR is r+ed or a try build is started
    # We don't want to run these always because they go towards
@ -95,25 +87,20 @@ matrix:
      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
  allow_failures:
    - os: windows
-      env: CARGO_INCREMENTAL=0 BASE_TESTS=true OS_WINDOWS=true
-  # prevent these jobs with default env vars
-  exclude:
-    - os: linux
-    - os: osx
-    - os: windows
+      env: CARGO_INCREMENTAL=0 OS_WINDOWS=true

 before_script:
  - |
-    if [ "$TRAVIS_BRANCH" == "auto" ] || [ "$TRAVIS_BRANCH" == "try" ]; then
-      pr=$(echo $TRAVIS_COMMIT_MESSAGE | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
-      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$pr" | \
+    if [[ "$TRAVIS_BRANCH" == "auto" ]] || [[ "$TRAVIS_BRANCH" == "try" ]]; then
+      PR=$(echo "$TRAVIS_COMMIT_MESSAGE" | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
+      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$PR" | \
        python -c "import sys, json; print(json.load(sys.stdin)['body'])" | \
        grep "^changelog: " | \
        sed "s/changelog: //g")
-      if [ -z "$output" ]; then
+      if [[ -z "$output" ]]; then
        echo "ERROR: PR body must contain 'changelog: ...'"
        exit 1
-      elif [ "$output" = "none" ]; then
+      elif [[ "$output" = "none" ]]; then
        echo "WARNING: changelog is 'none'"
      fi
    fi
@ -121,25 +108,30 @@ before_script:
    rm rust-toolchain
    ./setup-toolchain.sh
  - |
-    if [ "$TRAVIS_OS_NAME" == "windows" ]; then
-      export PATH=$PATH:$(rustc --print sysroot)/bin
-    else
-      export LD_LIBRARY_PATH=$(rustc --print sysroot)/lib
-    fi
+    SYSROOT=$(rustc --print sysroot)
+    case "$TRAVIS_OS_NAME" in
+      windows ) export PATH="${SYSROOT}/bin:${PATH}" ;;
+      linux ) export LD_LIBRARY_PATH="${SYSROOT}/lib" ;;
+      osx )
+        # See <https://github.com/nteract/nteract/issues/1523#issuecomment-301623519>
+        sudo mkdir -p /usr/local/lib
+        sudo find "$SYSROOT/lib" -maxdepth 1 -name '*.dylib' -exec ln -s {} /usr/local/lib \;
+        ;;
+    esac

 script:
  - |
-    if [ -z ${INTEGRATION} ]; then
-      travis_wait 30 ./ci/base-tests.sh && sleep 5
-    else
+    if [[ -n ${INTEGRATION} ]]; then
      ./ci/integration-tests.sh && sleep 5
+    else
+      ./ci/base-tests.sh && sleep 5
    fi

 after_success:
  - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
      set -e
-      if [ -z ${INTEGRATION} ]; then
+      if [[ -z ${INTEGRATION} ]]; then
        ./.github/deploy.sh
      else
        echo "Not deploying, because we're in an integration test run"
--- a/ci/base-tests.sh
+++ b/ci/base-tests.sh
@ -1,16 +1,20 @@
+#!/usr/bin/env bash
 set -ex

+# for faster build, share target dir between subcrates
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
+
 echo "Running clippy base tests"

 PATH=$PATH:./node_modules/.bin
-if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-  remark -f *.md -f doc/*.md > /dev/null
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  remark -f ./*.md -f doc/*.md > /dev/null
 fi
 # build clippy in debug mode and run tests
 cargo build --features "debugging deny-warnings"
 cargo test --features "debugging deny-warnings"
-# for faster build, share target dir between subcrates
-export CARGO_TARGET_DIR=`pwd`/target/
+
 (cd clippy_lints && cargo test)
 (cd rustc_tools_util && cargo test)
 (cd clippy_dev && cargo test)
@ -29,26 +33,27 @@ export CARGO_TARGET_DIR=`pwd`/target/
 (
  # Check sysroot handling
  sysroot=$(./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $(rustc --print sysroot)
+  test "$sysroot" = "$(rustc --print sysroot)"

-  if [ -z $OS_WINDOWS ]; then
+  if [[ -z "$OS_WINDOWS" ]]; then
    desired_sysroot=/tmp
  else
    desired_sysroot=C:/tmp
  fi
  sysroot=$(./target/debug/clippy-driver --sysroot $desired_sysroot --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot

  sysroot=$(SYSROOT=$desired_sysroot ./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot

  # Make sure this isn't set - clippy-driver should cope without it
  unset CARGO_MANIFEST_DIR

  # Run a lint and make sure it produces the expected output. It's also expected to exit with code 1
-  # XXX How to match the clippy invocation in compile-test.rs?
-  ! ./target/debug/clippy-driver -Dwarnings -Aunused -Zui-testing --emit metadata --crate-type bin tests/ui/cstring.rs 2> cstring.stderr
-  diff <(sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr) tests/ui/cstring.stderr
+  # FIXME: How to match the clippy invocation in compile-test.rs?
+  ./target/debug/clippy-driver -Dwarnings -Aunused -Zui-testing --emit metadata --crate-type bin tests/ui/cstring.rs 2> cstring.stderr && exit 1
+  sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr > normalized.stderr
+  diff normalized.stderr tests/ui/cstring.stderr

  # TODO: CLIPPY_CONF_DIR / CARGO_MANIFEST_DIR
 )
--- a/ci/integration-tests.sh
+++ b/ci/integration-tests.sh
@ -1,10 +1,13 @@
 #!/usr/bin/env bash
-set -x
+set -ex

 if [[ -z "$INTEGRATION" ]]; then
    exit 0
 fi

+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
+
 rm ~/.cargo/bin/cargo-clippy
 cargo install --force --debug --path .

--- a/tests/ui-toml/update-all-references.sh
+++ b/tests/ui-toml/update-all-references.sh
@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi

 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +
--- a/tests/ui-toml/update-references.sh
+++ b/tests/ui-toml/update-references.sh
@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
    echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi

-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")

 BUILD_DIR="$1"
 shift
@ -25,16 +25,14 @@ while [[ "$1" != "" ]]; do
    STDERR_NAME="${1/%.rs/.stderr}"
    STDOUT_NAME="${1/%.rs/.stdout}"
    shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
    fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
    fi
 done
-
-
--- a/tests/ui/update-all-references.sh
+++ b/tests/ui/update-all-references.sh
@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi

 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +
--- a/tests/ui/update-references.sh
+++ b/tests/ui/update-references.sh
@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
    echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi

-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")

 BUILD_DIR="$1"
 shift
@ -26,21 +26,19 @@ while [[ "$1" != "" ]]; do
    STDOUT_NAME="${1/%.rs/.stdout}"
    FIXED_NAME="${1/%.rs/.fixed}"
    shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
    fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
    fi
-    if [ -f $BUILD_DIR/$FIXED_NAME ] && \
-           ! (diff $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME >& /dev/null); then
-        echo updating $MYDIR/$FIXED_NAME
-        cp $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME
+    if [[ -f "$BUILD_DIR"/"$FIXED_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"); then
+        echo updating "$MYDIR"/"$FIXED_NAME"
+        cp "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"
    fi
 done
-
-
--- a/util/dev
+++ b/util/dev
@ -1,3 +1,5 @@
 #!/bin/sh
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR

-cd clippy_dev && cargo run -- $@
+cd clippy_dev && cargo run -- "$@"
--- a/util/fetch_prs_between.sh
+++ b/util/fetch_prs_between.sh
@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash

 # Fetches the merge commits between two git commits and prints the PR URL
 # together with the full commit message
@ -12,14 +12,15 @@ last=$2
 IFS='
 '
 for pr in $(git log --oneline --grep "Merge #" --grep "Merge pull request" --grep "Auto merge of" --grep "Rollup merge of" "$first...$last" | sort -rn | uniq); do
-  id=$(echo $pr | rg -o '#[0-9]{3,5}' | cut -c 2-)
-  commit=$(echo $pr | cut -d' ' -f 1)
-  message=$(git --no-pager show --pretty=medium $commit)
-  if [ ! -z $(echo "$message" | rg "^[\s]{4}changelog: [nN]one\.*$") ]; then
+  id=$(echo "$pr" | rg -o '#[0-9]{3,5}' | cut -c 2-)
+  commit=$(echo "$pr" | cut -d' ' -f 1)
+  message=$(git --no-pager show --pretty=medium "$commit")
+  if [[ -n $(echo "$message" | rg "^[\s]{4}changelog: [nN]one\.*$") ]]; then
    continue
  fi

  echo "URL: https://github.com/rust-lang/rust-clippy/pull/$id"
  echo "$message"
-  echo "---------------------------------------------------------\n"
+  echo "---------------------------------------------------------"
+  echo
 done