Stop using xargo for building the sysroot

It is currently broken. (see https://github.com/japaric/xargo/issues/227)

This makes it easier to for example patch whole crates away.

.gitignore

@@ -4,3 +4,6 @@ target
 *.o
 perf.data
 perf.data.old
+/build_sysroot/sysroot
+/build_sysroot/sysroot_src
+/build_sysroot/Cargo.lock

.travis.yml

@@ -10,11 +10,7 @@ rust:
   - nightly
 
 script:
-  - rustup component add rust-src
-  - cargo install xargo || echo "Skipping xargo install"
-  - cargo install hyperfine || echo "Skipping hyperfine install"
-  - ./prepare_libcore.sh
-  - ./test.sh
+  - ./prepare.sh && ./test.sh
 
 env:
   - RUST_BACKTRACE=1

alloc_system/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
-name = "alloc_system"
-version = "0.0.0"
-[lib]
-name = "alloc_system"
-path = "lib.rs"
-test = false
-doc = false
-[dependencies]
-core = { path = "../libcore" }
-libc = { path = "../rustc/libc_shim" }
-compiler_builtins = { path = "../rustc/compiler_builtins_shim" }
-# See comments in the source for what this dependency is
-[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))'.dependencies]
-dlmalloc = { path = "../rustc/dlmalloc_shim" }

build_sysroot/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+authors = ["bjorn3 <bjorn3@users.noreply.github.com>"]
+name = "sysroot"
+version = "0.0.0"
+
+[dependencies]
+core = { path = "./sysroot_src/src/libcore" }
+compiler_builtins = "0.1"
+alloc = { path = "./sysroot_src/src/liballoc" }
+
+alloc_system = { path = "./alloc_system" }
+
+[patch.crates-io]
+rustc-std-workspace-core = { path = "./sysroot_src/src/tools/rustc-std-workspace-core" }
+compiler_builtins = { path = "./compiler_builtins" }

build_sysroot/alloc_system/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
+name = "alloc_system"
+version = "0.0.0"
+[lib]
+name = "alloc_system"
+path = "lib.rs"
+test = false
+doc = false
+[dependencies]
+core = { path = "../sysroot_src/src/libcore" }
+libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false }
+compiler_builtins = "0.1"

build_sysroot/build_sysroot.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+cd $(dirname "$0")
+
+# Cleanup for previous run
+cargo clean
+rm Cargo.lock 2>/dev/null || true
+rm -r sysroot 2>/dev/null || true
+
+# FIXME find a better way to get the target triple
+unamestr=`uname`
+if [[ "$unamestr" == 'Linux' ]]; then
+   TARGET_TRIPLE='x86_64-unknown-linux-gnu'
+elif [[ "$unamestr" == 'Darwin' ]]; then
+   TARGET_TRIPLE='x86_64-apple-darwin'
+else
+   echo "Unsupported os"
+   exit 1
+fi
+
+# Build libs
+export RUSTFLAGS="$RUSTFLAGS -Z force-unstable-if-unmarked --sysroot ../"
+if [[ "$1" == "--release" ]]; then
+    channel='release'
+    RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target $TARGET_TRIPLE --release
+else
+    channel='debug'
+    cargo build --target $TARGET_TRIPLE
+fi
+
+# Copy files to sysroot
+mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+cp target/$TARGET_TRIPLE/$channel/deps/*.rlib sysroot/lib/rustlib/$TARGET_TRIPLE/lib/

build_sysroot/compiler_builtins/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "compiler_builtins"
+# Make sure the `compiler_builtins` from crates.io doesn't take precedence over this
+# replacement by specifying a higher version than the one on crates.io.
+version = "0.1.100"
+authors = ["bjorn3 <bjorn3@users.noreply.github.com>"]
+edition = "2018"
+
+[lib]
+name = "compiler_builtins"
+path = "lib.rs"
+test = false
+doc = false
+
+[dependencies]
+core = { path = "../sysroot_src/src/libcore" }
+
+[features]
+rustc-dep-of-std = []
+c = []

build_sysroot/compiler_builtins/lib.rs

@@ -0,0 +1,3 @@
+#![feature(compiler_builtins)]
+#![compiler_builtins]
+#![no_std]

build_sysroot/prepare_sysroot_src.sh

@@ -1,8 +1,9 @@
-#!/bin/bash --verbose
+#!/bin/bash
 set -e
+cd $(dirname "$0")
 
 SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/"
-DST_DIR="target/libcore"
+DST_DIR="sysroot_src"
 
 if [ ! -e $SRC_DIR ]; then
     echo "Please install rust-src component"
@@ -14,14 +15,17 @@ mkdir -p $DST_DIR/src
 cp -r $SRC_DIR/src $DST_DIR/
 
 pushd $DST_DIR
+echo "[GIT] init"
 git init
+echo "[GIT] add"
 git add .
+echo "[GIT] commit"
 git commit -m "Initial commit" -q
-git apply ../../patches/*.patch
+for file in $(ls ../../patches/ | grep -v patcha); do
+echo "[GIT] apply" $file
+git apply ../../patches/$file
+git commit -am "Patch $file"
+done
 popd
 
-# `alloc_system` has been merged with libstd, which doesn't build yet.
-# This copies the original source to the sysroot source dir to simplify building it
-cp -r alloc_system $DST_DIR/src/liballoc_system
-
 echo "Successfully prepared libcore for building"

build_sysroot/src/lib.rs

@@ -0,0 +1 @@
+#![no_std]

clean_all.sh

@@ -0,0 +1,4 @@
+#!/bin/bash --verbose
+set -e
+
+rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/,Cargo.lock} perf.data{,.old}

config.sh

@@ -19,5 +19,4 @@ else
 fi
 
 export RUSTFLAGS='-Zalways-encode-mir -Cpanic=abort -Zcodegen-backend='$(pwd)'/target/'$channel'/librustc_codegen_cranelift.'$dylib_ext
-export XARGO_RUST_SRC=$(pwd)'/target/libcore/src'
 RUSTC="rustc $RUSTFLAGS -L crate=target/out --out-dir target/out"

flamegraph.sh

@@ -5,7 +5,8 @@ source config.sh
 rm perf.data* || true
 
 # Profile compiling libcore
-perf record -F 9000 --call-graph dwarf -- $RUSTC --crate-type lib target/libcore/src/libcore/lib.rs --crate-name core
+perf record -F 9000 --call-graph dwarf \
+    -- $RUSTC --crate-type lib build_sysroot/sysroot_src/src/libcore/lib.rs --crate-name core
 
 # Generate the flamegraph
 perf script | ../FlameGraph/stackcollapse-perf.pl | grep cranelift | ../FlameGraph/flamegraph.pl > abc.svg

patches/0005-Disable-compiler_builtins.patch

@@ -1,29 +0,0 @@
-From d873311b7bd16bac383d71d49f93d0448cfcb0b1 Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Sun, 30 Sep 2018 18:32:50 +0200
-Subject: [PATCH] Disable compiler_builtins
-
----
- src/libcompiler_builtins/src/lib.rs | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/src/libcompiler_builtins/src/lib.rs b/src/libcompiler_builtins/src/lib.rs
-index 9f1dd15..b1eed26 100644
---- a/src/libcompiler_builtins/src/lib.rs
-+++ b/src/libcompiler_builtins/src/lib.rs
-@@ -34,6 +34,7 @@
- // that follow "x86 naming convention" (e.g. addsf3). Those aeabi intrinsics must adhere to the
- // AAPCS calling convention (`extern "aapcs"`) because that's how LLVM will call them.
- 
-+/*
- #[cfg(test)]
- extern crate core;
- 
-@@ -68,3 +69,4 @@ pub mod x86;
- pub mod x86_64;
- 
- pub mod probestack;
-+*/
--- 
-2.17.1 (Apple Git-112)
-

prepare.sh

@@ -0,0 +1,6 @@
+#!/bin/bash --verbose
+set -e
+
+rustup component add rust-src
+./build_sysroot/prepare_sysroot_src.sh
+cargo install hyperfine || echo "Skipping hyperfine install"

test.sh

@@ -21,26 +21,21 @@ SHOULD_RUN=1 $RUSTC --crate-type bin example/mini_core_hello_world.rs --cfg jit
 echo "[AOT] mini_core_hello_world"
 build_example_bin mini_core_hello_world example/mini_core_hello_world.rs
 
-pushd xargo
-rm -r ~/.xargo/HOST || true
-rm -r target || true
-time xargo build
-popd
+echo "[BUILD] sysroot"
+time ./build_sysroot/build_sysroot.sh
 
 # TODO linux linker doesn't accept duplicate definitions
-#$RUSTC --sysroot ~/.xargo/HOST example/alloc_example.rs --crate-type bin
+# echo "[BUILD+RUN] alloc_example"
+#$RUSTC --sysroot ./build_sysroot/sysroot example/alloc_example.rs --crate-type bin
 #./target/out/alloc_example
 
-$RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin
+echo "[BUILD] mod_bench"
+$RUSTC --sysroot ./build_sysroot/sysroot example/mod_bench.rs --crate-type bin
 
-echo "[BUILD] RUSTFLAGS=-Zmir-opt-level=3"
-pushd xargo
-rm -r ~/.xargo/HOST || true
-rm -r target || true
-time RUSTFLAGS="-Zmir-opt-level=3 $RUSTFLAGS" xargo build
-popd
+echo "[BUILD] sysroot in release mode"
+./build_sysroot/build_sysroot.sh --release
 
-COMPILE_MOD_BENCH_INLINE="$RUSTC --sysroot ~/.xargo/HOST example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -Og --crate-name mod_bench_inline"
+COMPILE_MOD_BENCH_INLINE="$RUSTC --sysroot ./build_sysroot/sysroot example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -Og --crate-name mod_bench_inline"
 COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o target/out/mod_bench_llvm_0 -Cpanic=abort"
 COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o target/out/mod_bench_llvm_1 -Cpanic=abort"
 COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o target/out/mod_bench_llvm_2 -Cpanic=abort"

xargo/Cargo.lock

@@ -1,4 +0,0 @@
-[[package]]
-name = "xargo"
-version = "0.1.0"
-

xargo/Cargo.toml

@@ -1,6 +0,0 @@
-[package]
-name = "xargo"
-version = "0.1.0"
-authors = ["bjorn3 <bjorn3@users.noreply.github.com>"]
-
-[dependencies]

xargo/Xargo.toml

@@ -1,7 +0,0 @@
-[dependencies]
-core = {}
-alloc = {}
-alloc_system = {}
-
-# Disabled due to missing support for many things (intrinsics, etc) in rustc_codegen_cranelift
-#std = {}

xargo/src/lib.rs

@@ -1,5 +0,0 @@
-#![feature(compiler_builtins)]
-
-#![no_std]
-#![compiler_builtins] // Don't link to compiler builtins
-// Just a dummy