OpenE2K
/
rust
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge commit '9a0c32934ebe376128230aa8da3275697b2053e7' into sync_cg_clif-2021-03-05

This commit is contained in:
bjorn3 2021-03-05 19:12:59 +01:00
parent 8fd946c63a 9a0c32934e
commit 7a6ea77473
73 changed files with 1145 additions and 2596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
compiler/rustc_codegen_cranelift/.cirrus.yml Normal file
View File

@ -0,0 +1,25 @@
task:
name: freebsd
freebsd_instance:
image: freebsd-12-1-release-amd64
setup_rust_script:
- pkg install -y curl git bash
- curl https://sh.rustup.rs -sSf --output rustup.sh
- sh rustup.sh --default-toolchain none -y --profile=minimal
cargo_bin_cache:
folder: ~/.cargo/bin
target_cache:
folder: target
prepare_script:
- . $HOME/.cargo/env
- git config --global user.email "user@example.com"
- git config --global user.name "User"
- ./prepare.sh
test_script:
- . $HOME/.cargo/env
- # Enable backtraces for easier debugging
- export RUST_BACKTRACE=1
- # Reduce amount of benchmark runs as they are slow
- export COMPILE_RUNS=2
- export RUN_RUNS=2
- ./test.sh

5
compiler/rustc_codegen_cranelift/.github/workflows/main.yml vendored
View File

@ -12,9 +12,6 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
env:
- BACKEND: ""
- BACKEND: --oldbe
steps:
- uses: actions/checkout@v2
@ -54,7 +51,7 @@ jobs:
export COMPILE_RUNS=2
export RUN_RUNS=2
./test.sh $BACKEND
./test.sh
- name: Package prebuilt cg_clif
run: tar cvfJ cg_clif.tar.xz build

2
compiler/rustc_codegen_cranelift/.vscode/settings.json vendored
View File

@ -1,6 +1,6 @@
{
// source for rustc_* is not included in the rust-src component; disable the errors about this
"rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"],
"rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
"rust-analyzer.assist.importMergeBehavior": "last",
"rust-analyzer.cargo.loadOutDirsFromCheck": true,
"rust-analyzer.linkedProjects": [

128
compiler/rustc_codegen_cranelift/Cargo.lock
View File

@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "anyhow"
version = "1.0.38"
@ -29,18 +31,6 @@ version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
[[package]]
name = "cc"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -49,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cranelift-bforest"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"cranelift-entity",
]
[[package]]
name = "cranelift-codegen"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"byteorder",
"cranelift-bforest",
@ -75,8 +65,8 @@ dependencies = [
[[package]]
name = "cranelift-codegen-meta"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"cranelift-codegen-shared",
"cranelift-entity",
@ -84,18 +74,18 @@ dependencies = [
[[package]]
name = "cranelift-codegen-shared"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
[[package]]
name = "cranelift-entity"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
[[package]]
name = "cranelift-frontend"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"cranelift-codegen",
"log",
@ -105,8 +95,8 @@ dependencies = [
[[package]]
name = "cranelift-jit"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -123,8 +113,8 @@ dependencies = [
[[package]]
name = "cranelift-module"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -135,18 +125,17 @@ dependencies = [
[[package]]
name = "cranelift-native"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"cranelift-codegen",
"raw-cpuid",
"target-lexicon",
]
[[package]]
name = "cranelift-object"
version = "0.69.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
version = "0.70.0"
source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -162,7 +151,7 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
]
[[package]]
@ -219,9 +208,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.82"
version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
[[package]]
name = "libloading"
@ -229,17 +218,17 @@ version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
dependencies = [
"cfg-if 1.0.0",
"cfg-if",
"winapi",
]
[[package]]
name = "log"
version = "0.4.13"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if 0.1.10",
"cfg-if",
]
[[package]]
@ -253,9 +242,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.22.0"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
dependencies = [
"crc32fast",
"indexmap",
@ -272,24 +261,13 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.8"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "raw-cpuid"
version = "8.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
dependencies = [
"bitflags",
"cc",
"rustc_version",
]
[[package]]
name = "regalloc"
version = "0.0.31"
@ -337,30 +315,6 @@ dependencies = [
"target-lexicon",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "smallvec"
version = "1.6.1"
@ -369,9 +323,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "syn"
version = "1.0.58"
version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
dependencies = [
"proc-macro2",
"quote",
@ -380,24 +334,24 @@ dependencies = [
[[package]]
name = "target-lexicon"
version = "0.11.1"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9"
checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95"
[[package]]
name = "thiserror"
version = "1.0.23"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.23"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
dependencies = [
"proc-macro2",
"quote",

5
compiler/rustc_codegen_cranelift/Cargo.toml
View File

@ -9,14 +9,14 @@ crate-type = ["dylib"]
[dependencies]
# These have to be in sync with each other
cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x86", "x64"] }
cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] }
cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
target-lexicon = "0.11.0"
gimli = { version = "0.23.0", default-features = false, features = ["write"]}
object = { version = "0.22.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
indexmap = "1.0.2"
@ -38,7 +38,6 @@ smallvec = "1.6.1"
default = ["jit", "inline_asm"]
jit = ["cranelift-jit", "libloading"]
inline_asm = []
oldbe = []
[profile.dev]
# By compiling dependencies with optimizations, performing tests gets much faster.

16
compiler/rustc_codegen_cranelift/build.sh
View File

@ -1,11 +1,10 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
# Settings
export CHANNEL="release"
build_sysroot="clif"
target_dir='build'
oldbe=''
while [[ $# != 0 ]]; do
case $1 in
"--debug")
@ -19,12 +18,9 @@ while [[ $# != 0 ]]; do
target_dir=$2
shift
;;
"--oldbe")
oldbe='--features oldbe'
;;
*)
echo "Unknown flag '$1'"
echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--oldbe]"
echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"
exit 1
;;
esac
@ -34,19 +30,19 @@ done
# Build cg_clif
unset CARGO_TARGET_DIR
unamestr=$(uname)
if [[ "$unamestr" == 'Linux' ]]; then
if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then
export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS
elif [[ "$unamestr" == 'Darwin' ]]; then
export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
dylib_ext='dylib'
else
echo "Unsupported os"
echo "Unsupported os $unamestr"
exit 1
fi
if [[ "$CHANNEL" == "release" ]]; then
cargo build $oldbe --release
cargo build --release
else
cargo build $oldbe
cargo build
fi
source scripts/ext_config.sh

20
compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
View File

@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "addr2line"
version = "0.14.1"
@ -30,15 +32,6 @@ dependencies = [
"core",
]
[[package]]
name = "alloc_system"
version = "0.0.0"
dependencies = [
"compiler_builtins",
"core",
"libc",
]
[[package]]
name = "autocfg"
version = "1.0.1"
@ -47,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "cc"
version = "1.0.66"
version = "1.0.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
[[package]]
name = "cfg-if"
@ -139,9 +132,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.84"
version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
dependencies = [
"rustc-std-workspace-core",
]
@ -258,7 +251,6 @@ name = "sysroot"
version = "0.0.0"
dependencies = [
"alloc",
"alloc_system",
"compiler_builtins",
"core",
"std",

2
compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml
View File

@ -9,8 +9,6 @@ alloc = { path = "./sysroot_src/library/alloc" }
std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
test = { path = "./sysroot_src/library/test" }
alloc_system = { path = "./alloc_system" }
compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
[patch.crates-io]

13
compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml
View File

@ -1,13 +0,0 @@
[package]
authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
name = "alloc_system"
version = "0.0.0"
[lib]
name = "alloc_system"
path = "lib.rs"
test = false
doc = false
[dependencies]
core = { path = "../sysroot_src/library/core" }
libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false }
compiler_builtins = "0.1"

2
compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
# Requires the CHANNEL env var to be set to `debug` or `release.`

4
compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"
@ -33,7 +33,7 @@ git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/
pushd compiler-builtins
git checkout -- .
git checkout 0.1.39
git apply ../../crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch
git apply ../../crate_patches/000*-compiler-builtins-*.patch
popd
echo "Successfully prepared sysroot source for building"

2
compiler/rustc_codegen_cranelift/clean_all.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash --verbose
#!/usr/bin/env bash
set -e
rm -rf target/ build/ build_sysroot/{sysroot_src/,target/,compiler-builtins/} perf.data{,.old}

48
compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch Normal file
View File

@ -0,0 +1,48 @@
From 1d574bf5e32d51641dcacaf8ef777e95b44f6f2a Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Thu, 18 Feb 2021 18:30:55 +0100
Subject: [PATCH] Disable 128bit atomic operations
Cranelift doesn't support them yet
---
src/mem/mod.rs | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/src/mem/mod.rs b/src/mem/mod.rs
index 107762c..2d1ae10 100644
--- a/src/mem/mod.rs
+++ b/src/mem/mod.rs
@@ -137,10 +137,6 @@ intrinsics! {
pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
memcpy_element_unordered_atomic(dest, src, bytes);
}
- #[cfg(target_has_atomic_load_store = "128")]
- pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
- memcpy_element_unordered_atomic(dest, src, bytes);
- }
#[cfg(target_has_atomic_load_store = "8")]
pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
@@ -158,10 +154,6 @@ intrinsics! {
pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
memmove_element_unordered_atomic(dest, src, bytes);
}
- #[cfg(target_has_atomic_load_store = "128")]
- pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
- memmove_element_unordered_atomic(dest, src, bytes);
- }
#[cfg(target_has_atomic_load_store = "8")]
pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
@@ -179,8 +171,4 @@ intrinsics! {
pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
memset_element_unordered_atomic(s, c, bytes);
}
- #[cfg(target_has_atomic_load_store = "128")]
- pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
- memset_element_unordered_atomic(s, c, bytes);
- }
}
--
2.26.2.7.g19db9cfb68

2
compiler/rustc_codegen_cranelift/example/alloc_example.rs
View File

@ -1,4 +1,4 @@
#![feature(start, box_syntax, alloc_system, core_intrinsics, alloc_prelude, alloc_error_handler)]
#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler)]
#![no_std]
extern crate alloc;

136
compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs → compiler/rustc_codegen_cranelift/example/alloc_system.rs
View File

@ -8,66 +8,24 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![no_std]
#![allow(unused_attributes)]
#![unstable(feature = "alloc_system",
reason = "this library is unlikely to be stabilized in its current \
form or name",
issue = "32838")]
#![feature(allocator_api)]
#![feature(core_intrinsics)]
#![feature(nll)]
#![feature(staged_api)]
#![feature(rustc_attrs)]
#![feature(alloc_layout_extra)]
#![cfg_attr(
all(target_arch = "wasm32", not(target_os = "emscripten")),
feature(integer_atomics, stdsimd)
)]
#![feature(allocator_api, rustc_private)]
#![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
// The minimum alignment guaranteed by the architecture. This value is used to
// add fast paths for low alignment values.
#[cfg(all(any(target_arch = "x86",
target_arch = "arm",
target_arch = "mips",
target_arch = "powerpc",
target_arch = "powerpc64",
target_arch = "asmjs",
target_arch = "wasm32")))]
#[allow(dead_code)]
target_arch = "powerpc64")))]
const MIN_ALIGN: usize = 8;
#[cfg(all(any(target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "mips64",
target_arch = "s390x",
target_arch = "sparc64")))]
#[allow(dead_code)]
const MIN_ALIGN: usize = 16;
/// The default memory allocator provided by the operating system.
///
/// This is based on `malloc` on Unix platforms and `HeapAlloc` on Windows,
/// plus related functions.
///
/// This type can be used in a `static` item
/// with the `#[global_allocator]` attribute
/// to force the global allocator to be the system’s one.
/// (The default is jemalloc for executables, on some platforms.)
///
/// ```rust
/// use std::alloc::System;
///
/// #[global_allocator]
/// static A: System = System;
///
/// fn main() {
/// let a = Box::new(4); // Allocates from the system allocator.
/// println!("{}", a);
/// }
/// ```
///
/// It can also be used directly to allocate memory
/// independently of the standard library’s global allocator.
#[stable(feature = "alloc_system_type", since = "1.28.0")]
pub struct System;
#[cfg(any(windows, unix, target_os = "redox"))]
mod realloc_fallback {
@ -96,7 +54,6 @@ mod platform {
use MIN_ALIGN;
use System;
use core::alloc::{GlobalAlloc, Layout};
#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@ -221,7 +178,6 @@ mod platform {
};
ptr as *mut u8
}
#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@ -254,89 +210,3 @@ mod platform {
}
}
}
// This is an implementation of a global allocator on the wasm32 platform when
// emscripten is not in use. In that situation there's no actual runtime for us
// to lean on for allocation, so instead we provide our own!
//
// The wasm32 instruction set has two instructions for getting the current
// amount of memory and growing the amount of memory. These instructions are the
// foundation on which we're able to build an allocator, so we do so! Note that
// the instructions are also pretty "global" and this is the "global" allocator
// after all!
//
// The current allocator here is the `dlmalloc` crate which we've got included
// in the rust-lang/rust repository as a submodule. The crate is a port of
// dlmalloc.c from C to Rust and is basically just so we can have "pure Rust"
// for now which is currently technically required (can't link with C yet).
//
// The crate itself provides a global allocator which on wasm has no
// synchronization as there are no threads!
#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
mod platform {
extern crate dlmalloc;
use core::alloc::{GlobalAlloc, Layout};
use System;
static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT;
#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
#[inline]
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let _lock = lock::lock();
DLMALLOC.malloc(layout.size(), layout.align())
}
#[inline]
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
let _lock = lock::lock();
DLMALLOC.calloc(layout.size(), layout.align())
}
#[inline]
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
let _lock = lock::lock();
DLMALLOC.free(ptr, layout.size(), layout.align())
}
#[inline]
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
let _lock = lock::lock();
DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size)
}
}
#[cfg(target_feature = "atomics")]
mod lock {
use core::arch::wasm32;
use core::sync::atomic::{AtomicI32, Ordering::SeqCst};
static LOCKED: AtomicI32 = AtomicI32::new(0);
pub struct DropLock;
pub fn lock() -> DropLock {
loop {
if LOCKED.swap(1, SeqCst) == 0 {
return DropLock
}
unsafe {
let r = wasm32::atomic::wait_i32(
&LOCKED as *const AtomicI32 as *mut i32,
1, // expected value
-1, // timeout
);
debug_assert!(r == 0 || r == 1);
}
}
}
impl Drop for DropLock {
fn drop(&mut self) {
let r = LOCKED.swap(0, SeqCst);
debug_assert_eq!(r, 1);
unsafe {
wasm32::atomic::wake(
&LOCKED as *const AtomicI32 as *mut i32,
1, // only one thread
);
}
}
}
}
#[cfg(not(target_feature = "atomics"))]
mod lock {
#[inline]
pub fn lock() {} // no atomics, no threads, that's easy!
}
}

31
compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs
View File

@ -1,22 +1,12 @@
// Adapted from rustc run-pass test suite
#![feature(no_core, arbitrary_self_types, box_syntax)]
#![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)]
#![feature(rustc_attrs)]
#![feature(start, lang_items)]
#![no_core]
extern crate mini_core;
use mini_core::*;
macro_rules! assert_eq {
($l:expr, $r: expr) => {
if $l != $r {
panic(stringify!($l != $r));
}
}
}
use std::{
ops::{Deref, CoerceUnsized, DispatchFromDyn},
marker::Unsize,
};
struct Ptr<T: ?Sized>(Box<T>);
@ -67,16 +57,13 @@ impl Trait for i32 {
}
}
#[start]
fn main(_: isize, _: *const *const u8) -> isize {
let pw = Ptr(box Wrapper(5)) as Ptr<Wrapper<dyn Trait>>;
fn main() {
let pw = Ptr(Box::new(Wrapper(5))) as Ptr<Wrapper<dyn Trait>>;
assert_eq!(pw.ptr_wrapper(), 5);
let wp = Wrapper(Ptr(box 6)) as Wrapper<Ptr<dyn Trait>>;
let wp = Wrapper(Ptr(Box::new(6))) as Wrapper<Ptr<dyn Trait>>;
assert_eq!(wp.wrapper_ptr(), 6);
let wpw = Wrapper(Ptr(box Wrapper(7))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
let wpw = Wrapper(Ptr(Box::new(Wrapper(7)))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
assert_eq!(wpw.wrapper_ptr_wrapper(), 7);
0
}

16
compiler/rustc_codegen_cranelift/example/mini_core.rs
View File

@ -365,6 +365,22 @@ impl <T: PartialEq> PartialEq for Option<T> {
}
}
#[lang = "shl"]
pub trait Shl<RHS = Self> {
type Output;
#[must_use]
fn shl(self, rhs: RHS) -> Self::Output;
}
impl Shl for u128 {
type Output = u128;
fn shl(self, rhs: u128) -> u128 {
self << rhs
}
}
#[lang = "neg"]
pub trait Neg {
type Output;

3
compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
View File

@ -264,6 +264,9 @@ fn main() {
assert_eq!(f2 as i8, -128);
assert_eq!(f2 as u8, 0);
let amount = 0;
assert_eq!(1u128 << amount, 1);
static ANOTHER_STATIC: &u8 = &A_STATIC;
assert_eq!(*ANOTHER_STATIC, 42);

16
compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch
View File

@ -119,21 +119,5 @@ index 6609bc3..241b497 100644
#[test]
#[should_panic(expected = "index 0 greater than length of slice")]
diff --git a/library/core/tests/num/ops.rs b/library/core/tests/num/ops.rs
index 9979cc8..d5d1d83 100644
--- a/library/core/tests/num/ops.rs
+++ b/library/core/tests/num/ops.rs
@@ -238,7 +238,7 @@ macro_rules! test_shift_assign {
}
};
}
-test_shift!(test_shl_defined, Shl::shl);
-test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
-test_shift!(test_shr_defined, Shr::shr);
-test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
+//test_shift!(test_shl_defined, Shl::shl);
+//test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
+//test_shift!(test_shr_defined, Shr::shr);
+//test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
--
2.21.0 (Apple Git-122)

103
compiler/rustc_codegen_cranelift/patches/0027-Disable-128bit-atomic-operations.patch Normal file
View File

@ -0,0 +1,103 @@
From 894e07dfec2624ba539129b1c1d63e1d7d812bda Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Thu, 18 Feb 2021 18:45:28 +0100
Subject: [PATCH] Disable 128bit atomic operations
Cranelift doesn't support them yet
---
library/core/src/sync/atomic.rs | 38 ---------------------------------
library/core/tests/atomic.rs | 4 ----
library/std/src/panic.rs | 6 ------
3 files changed, 48 deletions(-)
diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
index 81c9e1d..65c9503 100644
--- a/library/core/src/sync/atomic.rs
+++ b/library/core/src/sync/atomic.rs
@@ -2228,44 +2228,6 @@ atomic_int! {
"AtomicU64::new(0)",
u64 AtomicU64 ATOMIC_U64_INIT
}
-#[cfg(target_has_atomic_load_store = "128")]
-atomic_int! {
- cfg(target_has_atomic = "128"),
- cfg(target_has_atomic_equal_alignment = "128"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
- unstable(feature = "integer_atomics", issue = "32976"),
- "i128",
- "#![feature(integer_atomics)]\n\n",
- atomic_min, atomic_max,
- 16,
- "AtomicI128::new(0)",
- i128 AtomicI128 ATOMIC_I128_INIT
-}
-#[cfg(target_has_atomic_load_store = "128")]
-atomic_int! {
- cfg(target_has_atomic = "128"),
- cfg(target_has_atomic_equal_alignment = "128"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- unstable(feature = "integer_atomics", issue = "32976"),
- rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
- unstable(feature = "integer_atomics", issue = "32976"),
- "u128",
- "#![feature(integer_atomics)]\n\n",
- atomic_umin, atomic_umax,
- 16,
- "AtomicU128::new(0)",
- u128 AtomicU128 ATOMIC_U128_INIT
-}
macro_rules! atomic_int_ptr_sized {
( $($target_pointer_width:literal $align:literal)* ) => { $(
diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
index 2d1e449..cb6da5d 100644
--- a/library/core/tests/atomic.rs
+++ b/library/core/tests/atomic.rs
@@ -145,10 +145,6 @@ fn atomic_alignment() {
assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
#[cfg(target_has_atomic = "64")]
assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
- #[cfg(target_has_atomic = "128")]
- assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
- #[cfg(target_has_atomic = "128")]
- assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
#[cfg(target_has_atomic = "ptr")]
assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
#[cfg(target_has_atomic = "ptr")]
diff --git a/library/std/src/panic.rs b/library/std/src/panic.rs
index 89a822a..779fd88 100644
--- a/library/std/src/panic.rs
+++ b/library/std/src/panic.rs
@@ -279,9 +279,6 @@ impl RefUnwindSafe for atomic::AtomicI32 {}
#[cfg(target_has_atomic_load_store = "64")]
#[stable(feature = "integer_atomics_stable", since = "1.34.0")]
impl RefUnwindSafe for atomic::AtomicI64 {}
-#[cfg(target_has_atomic_load_store = "128")]
-#[unstable(feature = "integer_atomics", issue = "32976")]
-impl RefUnwindSafe for atomic::AtomicI128 {}
#[cfg(target_has_atomic_load_store = "ptr")]
#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
@@ -298,9 +295,6 @@ impl RefUnwindSafe for atomic::AtomicU32 {}
#[cfg(target_has_atomic_load_store = "64")]
#[stable(feature = "integer_atomics_stable", since = "1.34.0")]
impl RefUnwindSafe for atomic::AtomicU64 {}
-#[cfg(target_has_atomic_load_store = "128")]
-#[unstable(feature = "integer_atomics", issue = "32976")]
-impl RefUnwindSafe for atomic::AtomicU128 {}
#[cfg(target_has_atomic_load_store = "8")]
#[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
--
2.26.2.7.g19db9cfb68

2
compiler/rustc_codegen_cranelift/prepare.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash --verbose
#!/usr/bin/env bash
set -e
rustup component add rust-src rustc-dev llvm-tools-preview

2
compiler/rustc_codegen_cranelift/rust-toolchain
View File

@ -1 +1 @@
nightly-2021-01-30
nightly-2021-03-05

4
compiler/rustc_codegen_cranelift/rustfmt.toml Normal file
View File

@ -0,0 +1,4 @@
# Matches rustfmt.toml of rustc
version = "Two"
use_small_heuristics = "Max"
merge_derives = false

2
compiler/rustc_codegen_cranelift/scripts/cargo.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
dir=$(dirname "$0")
source "$dir/config.sh"

4
compiler/rustc_codegen_cranelift/scripts/config.sh
View File

@ -3,7 +3,7 @@
set -e
unamestr=$(uname)
if [[ "$unamestr" == 'Linux' ]]; then
if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
dylib_ext='so'
elif [[ "$unamestr" == 'Darwin' ]]; then
dylib_ext='dylib'
@ -26,7 +26,7 @@ export RUSTC=$dir"/bin/cg_clif"
export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
# FIXME remove once the atomic shim is gone
# FIXME fix `#[linkage = "extern_weak"]` without this
if [[ "$unamestr" == 'Darwin' ]]; then
export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
fi

2
compiler/rustc_codegen_cranelift/scripts/rustup.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e

42
compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/../"
@ -14,21 +14,18 @@ git checkout -- .
git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
git apply - <<EOF
diff --git a/.gitmodules b/.gitmodules
index 984113151de..c1e9d960d56 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -34,10 +34,6 @@
[submodule "src/doc/edition-guide"]
path = src/doc/edition-guide
url = https://github.com/rust-lang/edition-guide.git
-[submodule "src/llvm-project"]
- path = src/llvm-project
- url = https://github.com/rust-lang/llvm-project.git
- branch = rustc/11.0-2020-10-12
[submodule "src/doc/embedded-book"]
path = src/doc/embedded-book
url = https://github.com/rust-embedded/book.git
diff --git a/Cargo.toml b/Cargo.toml
index 5bd1147cad5..10d68a2ff14 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
+
[patch."https://github.com/rust-lang/rust-clippy"]
clippy_lints = { path = "src/tools/clippy/clippy_lints" }
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 23e689fcae7..5f077b765b6 100644
--- a/compiler/rustc_data_structures/Cargo.toml
@ -41,6 +38,19 @@ index 23e689fcae7..5f077b765b6 100644
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3", features = ["fileapi", "psapi"] }
diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
index d95b5b7f17f..00b6f0e3635 100644
--- a/library/alloc/Cargo.toml
+++ b/library/alloc/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2018"
[dependencies]
core = { path = "../core" }
-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
[dev-dependencies]
rand = "0.7"
EOF
cat > config.toml <<EOF

13
compiler/rustc_codegen_cranelift/scripts/tests.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
@ -27,13 +27,16 @@ function no_sysroot_tests() {
$MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target "$TARGET_TRIPLE"
$RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
# (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
$MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
$RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
}
function base_sysroot_tests() {
echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
$MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
$RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
echo "[AOT] alloc_system"
$MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
echo "[AOT] alloc_example"
$MY_RUSTC example/alloc_example.rs --crate-type bin --target "$TARGET_TRIPLE"
$RUN_WRAPPER ./target/out/alloc_example

56
compiler/rustc_codegen_cranelift/src/abi/comments.rs
View File

@ -10,14 +10,14 @@ use cranelift_codegen::entity::EntityRef;
use crate::prelude::*;
pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
fx.add_global_comment(
"kind loc.idx param pass mode ty".to_string(),
);
}
pub(super) fn add_arg_comment<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
kind: &str,
local: Option<mir::Local>,
local_field: Option<usize>,
@ -42,11 +42,7 @@ pub(super) fn add_arg_comment<'tcx>(
[param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)),
params => Cow::Owned(format!(
"= {}",
params
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(",")
params.iter().map(ToString::to_string).collect::<Vec<_>>().join(",")
)),
};
@ -62,7 +58,7 @@ pub(super) fn add_arg_comment<'tcx>(
));
}
pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
fx.add_global_comment(String::new());
fx.add_global_comment(
"kind local ty size align (abi,pref)".to_string(),
@ -70,19 +66,13 @@ pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>
}
pub(super) fn add_local_place_comments<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
place: CPlace<'tcx>,
local: Local,
) {
let TyAndLayout { ty, layout } = place.layout();
let rustc_target::abi::Layout {
size,
align,
abi: _,
variants: _,
fields: _,
largest_niche: _,
} = layout;
let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
layout;
let (kind, extra) = match *place.inner() {
CPlaceInner::Var(place_local, var) => {
@ -91,10 +81,7 @@ pub(super) fn add_local_place_comments<'tcx>(
}
CPlaceInner::VarPair(place_local, var1, var2) => {
assert_eq!(local, place_local);
(
"ssa",
Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())),
)
("ssa", Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())))
}
CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(),
CPlaceInner::Addr(ptr, meta) => {
@ -104,18 +91,15 @@ pub(super) fn add_local_place_comments<'tcx>(
Cow::Borrowed("")
};
match ptr.base_and_offset() {
(crate::pointer::PointerBase::Addr(addr), offset) => (
"reuse",
format!("storage={}{}{}", addr, offset, meta).into(),
),
(crate::pointer::PointerBase::Stack(stack_slot), offset) => (
"stack",
format!("storage={}{}{}", stack_slot, offset, meta).into(),
),
(crate::pointer::PointerBase::Dangling(align), offset) => (
"zst",
format!("align={},offset={}", align.bytes(), offset).into(),
),
(crate::pointer::PointerBase::Addr(addr), offset) => {
("reuse", format!("storage={}{}{}", addr, offset, meta).into())
}
(crate::pointer::PointerBase::Stack(stack_slot), offset) => {
("stack", format!("storage={}{}{}", stack_slot, offset, meta).into())
}
(crate::pointer::PointerBase::Dangling(align), offset) => {
("zst", format!("align={},offset={}", align.bytes(), offset).into())
}
}
}
};
@ -128,11 +112,7 @@ pub(super) fn add_local_place_comments<'tcx>(
size.bytes(),
align.abi.bytes(),
align.pref.bytes(),
if extra.is_empty() {
""
} else {
" "
},
if extra.is_empty() { "" } else { " " },
extra,
));
}

144
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View File

@ -38,25 +38,15 @@ fn clif_sig_from_fn_abi<'tcx>(
| Conv::X86VectorCall
| Conv::AmdGpuKernel
| Conv::AvrInterrupt
| Conv::AvrNonBlockingInterrupt => {
todo!("{:?}", fn_abi.conv)
}
| Conv::AvrNonBlockingInterrupt => todo!("{:?}", fn_abi.conv),
};
let inputs = fn_abi
.args
.iter()
.map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter())
.flatten();
let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
// Sometimes the first param is an pointer to the place where the return value needs to be stored.
let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
Signature {
params,
returns,
call_conv,
}
Signature { params, returns, call_conv }
}
pub(crate) fn get_function_sig<'tcx>(
@ -65,34 +55,25 @@ pub(crate) fn get_function_sig<'tcx>(
inst: Instance<'tcx>,
) -> Signature {
assert!(!inst.substs.needs_infer());
clif_sig_from_fn_abi(
tcx,
triple,
&FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]),
)
clif_sig_from_fn_abi(tcx, triple, &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]))
}
/// Instance must be monomorphized
pub(crate) fn import_function<'tcx>(
tcx: TyCtxt<'tcx>,
module: &mut impl Module,
module: &mut dyn Module,
inst: Instance<'tcx>,
) -> FuncId {
let name = tcx.symbol_name(inst).name.to_string();
let sig = get_function_sig(tcx, module.isa().triple(), inst);
module
.declare_function(&name, Linkage::Import, &sig)
.unwrap()
module.declare_function(&name, Linkage::Import, &sig).unwrap()
}
impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
impl<'tcx> FunctionCx<'_, '_, 'tcx> {
/// Instance must be monomorphized
pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
let func_id = import_function(self.tcx, &mut self.cx.module, inst);
let func_ref = self
.cx
.module
.declare_func_in_func(func_id, &mut self.bcx.func);
let func_id = import_function(self.tcx, self.cx.module, inst);
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
#[cfg(debug_assertions)]
self.add_comment(func_ref, format!("{:?}", inst));
@ -107,20 +88,9 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
returns: Vec<AbiParam>,
args: &[Value],
) -> &[Value] {
let sig = Signature {
params,
returns,
call_conv: CallConv::triple_default(self.triple()),
};
let func_id = self
.cx
.module
.declare_function(&name, Linkage::Import, &sig)
.unwrap();
let func_ref = self
.cx
.module
.declare_func_in_func(func_id, &mut self.bcx.func);
let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
let call_inst = self.bcx.ins().call(func_ref, args);
#[cfg(debug_assertions)]
{
@ -140,17 +110,12 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
let (input_tys, args): (Vec<_>, Vec<_>) = args
.iter()
.map(|arg| {
(
AbiParam::new(self.clif_type(arg.layout().ty).unwrap()),
arg.load_scalar(self),
)
(AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
})
.unzip();
let return_layout = self.layout_of(return_ty);
let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
tup.types()
.map(|ty| AbiParam::new(self.clif_type(ty).unwrap()))
.collect()
tup.types().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
} else {
vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
};
@ -169,7 +134,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
/// Make a [`CPlace`] capable of holding value of the specified type.
fn make_local_place<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
local: Local,
layout: TyAndLayout<'tcx>,
is_ssa: bool,
@ -190,10 +155,7 @@ fn make_local_place<'tcx>(
place
}
pub(crate) fn codegen_fn_prelude<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
start_block: Block,
) {
pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) {
fx.bcx.append_block_params_for_function_params(start_block);
fx.bcx.switch_to_block(start_block);
@ -204,13 +166,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
#[cfg(debug_assertions)]
self::comments::add_args_header_comment(fx);
let mut block_params_iter = fx
.bcx
.func
.dfg
.block_params(start_block)
.to_vec()
.into_iter();
let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
let ret_place =
self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
@ -286,10 +242,10 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
if let Some((addr, meta)) = val.try_to_ptr() {
let local_decl = &fx.mir.local_decls[local];
// v this ! is important
let internally_mutable = !val.layout().ty.is_freeze(
fx.tcx.at(local_decl.source_info.span),
ParamEnv::reveal_all(),
);
let internally_mutable = !val
.layout()
.ty
.is_freeze(fx.tcx.at(local_decl.source_info.span), ParamEnv::reveal_all());
if local_decl.mutability == mir::Mutability::Not && !internally_mutable {
// We wont mutate this argument, so it is fine to borrow the backing storage
// of this argument, to prevent a copy.
@ -321,9 +277,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
ArgKind::Spread(params) => {
for (i, param) in params.into_iter().enumerate() {
if let Some(param) = param {
place
.place_field(fx, mir::Field::new(i))
.write_cvalue(fx, param);
place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
}
}
}
@ -340,13 +294,11 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
assert_eq!(fx.local_map.push(place), local);
}
fx.bcx
.ins()
.jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
}
pub(crate) fn codegen_terminator_call<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
span: Span,
current_block: Block,
func: &Operand<'tcx>,
@ -354,9 +306,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
destination: Option<(Place<'tcx>, BasicBlock)>,
) {
let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
let fn_sig = fx
.tcx
.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
let fn_sig =
fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
let destination = destination.map(|(place, bb)| (codegen_place(fx, place), bb));
@ -404,20 +355,11 @@ pub(crate) fn codegen_terminator_call<'tcx>(
let fn_abi = if let Some(instance) = instance {
FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
} else {
FnAbi::of_fn_ptr(
&RevealAllLayoutCx(fx.tcx),
fn_ty.fn_sig(fx.tcx),
&extra_args,
)
FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
};
let is_cold = instance
.map(|inst| {
fx.tcx
.codegen_fn_attrs(inst.def_id())
.flags
.contains(CodegenFnAttrFlags::COLD)
})
.map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
.unwrap_or(false);
if is_cold {
fx.cold_blocks.insert(current_block);
@ -441,9 +383,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
}
args
} else {
args.iter()
.map(|arg| codegen_operand(fx, arg))
.collect::<Vec<_>>()
args.iter().map(|arg| codegen_operand(fx, arg)).collect::<Vec<_>>()
};
// | indirect call target
@ -451,10 +391,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
// v v
let (func_ref, first_arg) = match instance {
// Trait object call
Some(Instance {
def: InstanceDef::Virtual(_, idx),
..
}) => {
Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
#[cfg(debug_assertions)]
{
let nop_inst = fx.bcx.ins().nop();
@ -511,10 +448,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
)
.collect::<Vec<_>>();
if instance
.map(|inst| inst.def.requires_caller_location(fx.tcx))
.unwrap_or(false)
{
if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
// Pass the caller location for `#[track_caller]`.
let caller_location = fx.get_caller_location(span);
call_args.extend(
@ -543,10 +477,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
// FIXME find a cleaner way to support varargs
if fn_sig.c_variadic {
if fn_sig.abi != Abi::C {
fx.tcx.sess.span_fatal(
span,
&format!("Variadic call for non-C abi {:?}", fn_sig.abi),
);
fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
}
let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
let abi_params = call_args
@ -555,9 +486,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
let ty = fx.bcx.func.dfg.value_type(arg);
if !ty.is_int() {
// FIXME set %al to upperbound on float args once floats are supported
fx.tcx
.sess
.span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
fx.tcx.sess.span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
}
AbiParam::new(ty)
})
@ -574,7 +503,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
}
pub(crate) fn codegen_drop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
span: Span,
drop_place: CPlace<'tcx>,
) {
@ -611,10 +540,7 @@ pub(crate) fn codegen_drop<'tcx>(
fx,
fx.layout_of(fx.tcx.mk_ref(
&ty::RegionKind::ReErased,
TypeAndMut {
ty,
mutbl: crate::rustc_hir::Mutability::Mut,
},
TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
)),
);
let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]);

108
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View File

@ -71,12 +71,7 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
.prefix
.iter()
.flatten()
.map(|&kind| {
reg_to_abi_param(Reg {
kind,
size: cast.prefix_chunk_size,
})
})
.map(|&kind| reg_to_abi_param(Reg { kind, size: cast.prefix_chunk_size }))
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
.collect::<SmallVec<_>>();
@ -98,12 +93,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
match self.mode {
PassMode::Ignore => smallvec![],
PassMode::Direct(attrs) => match &self.layout.abi {
Abi::Scalar(scalar) => {
smallvec![apply_arg_attrs_to_abi_param(
AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
attrs
)]
}
Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param(
AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
attrs
)],
Abi::Vector { .. } => {
let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
smallvec![AbiParam::new(vector_ty)]
@ -122,11 +115,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
_ => unreachable!("{:?}", self.layout.abi),
},
PassMode::Cast(cast) => cast_target_to_abi_params(cast),
PassMode::Indirect {
attrs,
extra_attrs: None,
on_stack,
} => {
PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
if on_stack {
let size = u32::try_from(self.layout.size.bytes()).unwrap();
smallvec![apply_arg_attrs_to_abi_param(
@ -134,17 +123,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
attrs
)]
} else {
smallvec![apply_arg_attrs_to_abi_param(
AbiParam::new(pointer_ty(tcx)),
attrs
)]
smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
}
}
PassMode::Indirect {
attrs,
extra_attrs: Some(extra_attrs),
on_stack,
} => {
PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => {
assert!(!on_stack);
smallvec![
apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
@ -158,10 +140,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
match self.mode {
PassMode::Ignore => (None, vec![]),
PassMode::Direct(_) => match &self.layout.abi {
Abi::Scalar(scalar) => (
None,
vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))],
),
Abi::Scalar(scalar) => {
(None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))])
}
Abi::Vector { .. } => {
let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
(None, vec![AbiParam::new(vector_ty)])
@ -177,31 +158,19 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
_ => unreachable!("{:?}", self.layout.abi),
},
PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack,
} => {
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => {
assert!(!on_stack);
(
Some(AbiParam::special(
pointer_ty(tcx),
ArgumentPurpose::StructReturn,
)),
vec![],
)
(Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
}
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
}
}
}
pub(super) fn to_casted_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
arg: CValue<'tcx>,
cast: CastTarget,
) -> SmallVec<[Value; 2]> {
@ -211,9 +180,7 @@ pub(super) fn to_casted_value<'tcx>(
cast_target_to_abi_params(cast)
.into_iter()
.map(|param| {
let val = ptr
.offset_i64(fx, offset)
.load(fx, param.value_type, MemFlags::new());
let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
offset += i64::from(param.value_type.bytes());
val
})
@ -221,16 +188,13 @@ pub(super) fn to_casted_value<'tcx>(
}
pub(super) fn from_casted_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
block_params: &[Value],
layout: TyAndLayout<'tcx>,
cast: CastTarget,
) -> CValue<'tcx> {
let abi_params = cast_target_to_abi_params(cast);
let abi_param_size: u32 = abi_params
.iter()
.map(|param| param.value_type.bytes())
.sum();
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
@ -260,7 +224,7 @@ pub(super) fn from_casted_value<'tcx>(
/// Get a set of values to be passed as function arguments.
pub(super) fn adjust_arg_for_abi<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
arg: CValue<'tcx>,
arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
) -> SmallVec<[Value; 2]> {
@ -283,7 +247,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
/// Create a [`CValue`] containing the value of a function parameter adding clif function parameters
/// as necessary.
pub(super) fn cvalue_for_param<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
@ -294,10 +258,7 @@ pub(super) fn cvalue_for_param<'tcx>(
.into_iter()
.map(|abi_param| {
let block_param = block_params_iter.next().unwrap();
assert_eq!(
fx.bcx.func.dfg.value_type(block_param),
abi_param.value_type
);
assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type);
block_param
})
.collect::<SmallVec<[_; 2]>>();
@ -321,29 +282,14 @@ pub(super) fn cvalue_for_param<'tcx>(
}
PassMode::Pair(_, _) => {
assert_eq!(block_params.len(), 2, "{:?}", block_params);
Some(CValue::by_val_pair(
block_params[0],
block_params[1],
arg_abi.layout,
))
Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout))
}
PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
assert_eq!(block_params.len(), 1, "{:?}", block_params);
Some(CValue::by_ref(
Pointer::new(block_params[0]),
arg_abi.layout,
))
Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout))
}
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => {
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
assert_eq!(block_params.len(), 2, "{:?}", block_params);
Some(CValue::by_ref_unsized(
Pointer::new(block_params[0]),

90
compiler/rustc_codegen_cranelift/src/abi/returning.rs
View File

@ -8,14 +8,13 @@ use smallvec::{smallvec, SmallVec};
/// Can the given type be returned into an ssa var or does it need to be returned on the stack.
pub(crate) fn can_return_to_ssa_var<'tcx>(
fx: &FunctionCx<'_, 'tcx, impl Module>,
fx: &FunctionCx<'_, '_, 'tcx>,
func: &mir::Operand<'tcx>,
args: &[mir::Operand<'tcx>],
) -> bool {
let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
let fn_sig = fx
.tcx
.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
let fn_sig =
fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
// Handle special calls like instrinsics and empty drop glue.
let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
@ -42,11 +41,7 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
let fn_abi = if let Some(instance) = instance {
FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
} else {
FnAbi::of_fn_ptr(
&RevealAllLayoutCx(fx.tcx),
fn_ty.fn_sig(fx.tcx),
&extra_args,
)
FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
};
match fn_abi.ret.mode {
PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true,
@ -58,15 +53,12 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
/// Return a place where the return value of the current function can be written to. If necessary
/// this adds an extra parameter pointing to where the return value needs to be stored.
pub(super) fn codegen_return_param<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>,
block_params_iter: &mut impl Iterator<Item = Value>,
) -> CPlace<'tcx> {
let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
PassMode::Ignore => (
CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout),
smallvec![],
),
PassMode::Ignore => (CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), smallvec![]),
PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
(
@ -79,26 +71,17 @@ pub(super) fn codegen_return_param<'tcx>(
smallvec![],
)
}
PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
let ret_param = block_params_iter.next().unwrap();
assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx));
(
CPlace::for_ptr(
Pointer::new(ret_param),
fx.fn_abi.as_ref().unwrap().ret.layout,
),
CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.as_ref().unwrap().ret.layout),
smallvec![ret_param],
)
}
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
};
#[cfg(not(debug_assertions))]
@ -120,27 +103,21 @@ pub(super) fn codegen_return_param<'tcx>(
/// Invokes the closure with if necessary a value representing the return pointer. When the closure
/// returns the call return value(s) if any are written to the correct place.
pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
fx: &mut FunctionCx<'_, 'tcx, M>,
pub(super) fn codegen_with_call_return_arg<'tcx, T>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
ret_place: Option<CPlace<'tcx>>,
f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option<Value>) -> (Inst, T),
f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> (Inst, T),
) -> (Inst, T) {
let return_ptr = match ret_arg_abi.mode {
PassMode::Ignore => None,
PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => match ret_place {
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => match ret_place {
Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)),
None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot
},
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None,
};
@ -177,37 +154,24 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
ret_place.write_cvalue(fx, result);
}
}
PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {}
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {}
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
}
(call_inst, meta)
}
/// Codegen a return instruction with the right return value(s) if any.
pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) {
pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
match fx.fn_abi.as_ref().unwrap().ret.mode {
PassMode::Ignore
| PassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {
PassMode::Ignore | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
fx.bcx.ins().return_(&[]);
}
PassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
PassMode::Direct(_) => {
let place = fx.get_local_place(RETURN_PLACE);
let ret_val = place.to_cvalue(fx).load_scalar(fx);

29
compiler/rustc_codegen_cranelift/src/allocator.rs
View File

@ -66,13 +66,9 @@ fn codegen_inner(
let callee_name = kind.fn_name(method.name);
//eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
let func_id = module
.declare_function(&caller_name, Linkage::Export, &sig)
.unwrap();
let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
let callee_func_id = module
.declare_function(&callee_name, Linkage::Import, &sig)
.unwrap();
let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone());
@ -96,11 +92,7 @@ fn codegen_inner(
bcx.finalize();
}
module
.define_function(
func_id,
&mut ctx,
&mut cranelift_codegen::binemit::NullTrapSink {},
)
.define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
.unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
}
@ -114,13 +106,10 @@ fn codegen_inner(
let callee_name = kind.fn_name(sym::oom);
//eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
let func_id = module
.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig)
.unwrap();
let func_id =
module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
let callee_func_id = module
.declare_function(&callee_name, Linkage::Import, &sig)
.unwrap();
let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
@ -143,11 +132,7 @@ fn codegen_inner(
bcx.finalize();
}
module
.define_function(
func_id,
&mut ctx,
&mut cranelift_codegen::binemit::NullTrapSink {},
)
.define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
.unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
}

9
compiler/rustc_codegen_cranelift/src/analyze.rs
View File

@ -11,7 +11,7 @@ pub(crate) enum SsaKind {
Ssa,
}
pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, SsaKind> {
pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
let mut flag_map = fx
.mir
.local_decls
@ -40,12 +40,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, S
}
match &bb.terminator().kind {
TerminatorKind::Call {
destination,
func,
args,
..
} => {
TerminatorKind::Call { destination, func, args, .. } => {
if let Some((dest_place, _dest_bb)) = destination {
if !crate::abi::can_return_to_ssa_var(fx, func, args) {
not_ssa(&mut flag_map, dest_place.local)

49
compiler/rustc_codegen_cranelift/src/archive.rs
View File

@ -12,10 +12,7 @@ use object::{Object, ObjectSymbol, SymbolKind};
#[derive(Debug)]
enum ArchiveEntry {
FromArchive {
archive_index: usize,
entry_index: usize,
},
FromArchive { archive_index: usize, entry_index: usize },
File(PathBuf),
}
@ -30,7 +27,6 @@ pub(crate) struct ArArchiveBuilder<'a> {
// Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
// the end of an archive for linkers to not get confused.
entries: Vec<(String, ArchiveEntry)>,
update_symbols: bool,
}
impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
@ -46,10 +42,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
let entry = entry.unwrap();
entries.push((
String::from_utf8(entry.header().identifier().to_vec()).unwrap(),
ArchiveEntry::FromArchive {
archive_index: 0,
entry_index: i,
},
ArchiveEntry::FromArchive { archive_index: 0, entry_index: i },
));
i += 1;
}
@ -69,7 +62,6 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
src_archives,
entries,
update_symbols: false,
}
}
@ -95,14 +87,9 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) {
let location = find_library(name, &self.lib_search_paths, self.sess);
self.add_archive(location.clone(), |_| false)
.unwrap_or_else(|e| {
panic!(
"failed to add native library {}: {}",
location.to_string_lossy(),
e
);
});
self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| {
panic!("failed to add native library {}: {}", location.to_string_lossy(), e);
});
}
fn add_rlib(
@ -136,9 +123,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
})
}
fn update_symbols(&mut self) {
self.update_symbols = true;
}
fn update_symbols(&mut self) {}
fn build(mut self) {
enum BuilderKind {
@ -156,10 +141,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
// FIXME only read the symbol table of the object files to avoid having to keep all
// object files in memory at once, or read them twice.
let data = match entry {
ArchiveEntry::FromArchive {
archive_index,
entry_index,
} => {
ArchiveEntry::FromArchive { archive_index, entry_index } => {
// FIXME read symbols from symtab
use std::io::Read;
let (ref _src_archive_path, ref mut src_archive) =
@ -225,10 +207,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
err
));
}),
entries
.iter()
.map(|(name, _)| name.as_bytes().to_vec())
.collect(),
entries.iter().map(|(name, _)| name.as_bytes().to_vec()).collect(),
ar::GnuSymbolTableFormat::Size32,
symbol_table,
)
@ -271,8 +250,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
.expect("Couldn't run ranlib");
if !status.success() {
self.sess
.fatal(&format!("Ranlib exited with code {:?}", status.code()));
self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
}
}
}
@ -292,13 +270,8 @@ impl<'a> ArArchiveBuilder<'a> {
let file_name = String::from_utf8(entry.header().identifier().to_vec())
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
if !skip(&file_name) {
self.entries.push((
file_name,
ArchiveEntry::FromArchive {
archive_index,
entry_index: i,
},
));
self.entries
.push((file_name, ArchiveEntry::FromArchive { archive_index, entry_index: i }));
}
i += 1;
}

185
compiler/rustc_codegen_cranelift/src/atomic_shim.rs
View File

@ -1,185 +0,0 @@
//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support
//! atomic operations yet.
// FIXME implement atomic instructions in Cranelift.
use crate::prelude::*;
#[cfg(all(feature = "jit", unix))]
#[no_mangle]
static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
pub(crate) fn init_global_lock(
module: &mut impl Module,
bcx: &mut FunctionBuilder<'_>,
use_jit: bool,
) {
if use_jit {
// When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here,
// so instead we define it in the cg_clif dylib.
return;
}
let mut data_ctx = DataContext::new();
data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms.
data_ctx.set_align(16);
let atomic_mutex = module
.declare_data(
"__cg_clif_global_atomic_mutex",
Linkage::Export,
true,
false,
)
.unwrap();
module.define_data(atomic_mutex, &data_ctx).unwrap();
let pthread_mutex_init = module
.declare_function(
"pthread_mutex_init",
Linkage::Import,
&cranelift_codegen::ir::Signature {
call_conv: module.target_config().default_call_conv,
params: vec![
AbiParam::new(
module.target_config().pointer_type(), /* *mut pthread_mutex_t */
),
AbiParam::new(
module.target_config().pointer_type(), /* *const pthread_mutex_attr_t */
),
],
returns: vec![AbiParam::new(types::I32 /* c_int */)],
},
)
.unwrap();
let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func);
let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func);
let atomic_mutex = bcx
.ins()
.global_value(module.target_config().pointer_type(), atomic_mutex);
let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0);
bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]);
}
pub(crate) fn init_global_lock_constructor(
module: &mut impl Module,
constructor_name: &str,
) -> FuncId {
let sig = Signature::new(CallConv::SystemV);
let init_func_id = module
.declare_function(constructor_name, Linkage::Export, &sig)
.unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
let block = bcx.create_block();
bcx.switch_to_block(block);
crate::atomic_shim::init_global_lock(module, &mut bcx, false);
bcx.ins().return_(&[]);
bcx.seal_all_blocks();
bcx.finalize();
}
module
.define_function(
init_func_id,
&mut ctx,
&mut cranelift_codegen::binemit::NullTrapSink {},
)
.unwrap();
init_func_id
}
pub(crate) fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
let atomic_mutex = fx
.cx
.module
.declare_data(
"__cg_clif_global_atomic_mutex",
Linkage::Import,
true,
false,
)
.unwrap();
let pthread_mutex_lock = fx
.cx
.module
.declare_function(
"pthread_mutex_lock",
Linkage::Import,
&cranelift_codegen::ir::Signature {
call_conv: fx.cx.module.target_config().default_call_conv,
params: vec![AbiParam::new(
fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
)],
returns: vec![AbiParam::new(types::I32 /* c_int */)],
},
)
.unwrap();
let pthread_mutex_lock = fx
.cx
.module
.declare_func_in_func(pthread_mutex_lock, fx.bcx.func);
let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
let atomic_mutex = fx
.bcx
.ins()
.global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]);
}
pub(crate) fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
let atomic_mutex = fx
.cx
.module
.declare_data(
"__cg_clif_global_atomic_mutex",
Linkage::Import,
true,
false,
)
.unwrap();
let pthread_mutex_unlock = fx
.cx
.module
.declare_function(
"pthread_mutex_unlock",
Linkage::Import,
&cranelift_codegen::ir::Signature {
call_conv: fx.cx.module.target_config().default_call_conv,
params: vec![AbiParam::new(
fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
)],
returns: vec![AbiParam::new(types::I32 /* c_int */)],
},
)
.unwrap();
let pthread_mutex_unlock = fx
.cx
.module
.declare_func_in_func(pthread_mutex_unlock, fx.bcx.func);
let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
let atomic_mutex = fx
.bcx
.ins()
.global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]);
}

62
compiler/rustc_codegen_cranelift/src/backend.rs
View File

@ -8,7 +8,7 @@ use rustc_session::Session;
use cranelift_module::FuncId;
use object::write::*;
use object::{RelocationEncoding, RelocationKind, SectionKind, SymbolFlags};
use object::{RelocationEncoding, SectionKind, SymbolFlags};
use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
@ -22,9 +22,7 @@ pub(crate) trait WriteMetadata {
impl WriteMetadata for object::write::Object {
fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) {
let segment = self
.segment_name(object::write::StandardSegment::Data)
.to_vec();
let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
let offset = self.append_section_data(section_id, &data, 1);
// For MachO and probably PE this is necessary to prevent the linker from throwing away the
@ -74,11 +72,7 @@ impl WriteDebugInfo for ObjectProduct {
let section_id = self.object.add_section(
segment,
name,
if id == SectionId::EhFrame {
SectionKind::ReadOnlyData
} else {
SectionKind::Debug
},
if id == SectionId::EhFrame { SectionKind::ReadOnlyData } else { SectionKind::Debug },
);
self.object
.section_mut(section_id)
@ -118,49 +112,6 @@ impl WriteDebugInfo for ObjectProduct {
}
}
// FIXME remove once atomic instructions are implemented in Cranelift.
pub(crate) trait AddConstructor {
fn add_constructor(&mut self, func_id: FuncId);
}
impl AddConstructor for ObjectProduct {
fn add_constructor(&mut self, func_id: FuncId) {
let symbol = self.function_symbol(func_id);
let segment = self
.object
.segment_name(object::write::StandardSegment::Data);
let init_array_section =
self.object
.add_section(segment.to_vec(), b".init_array".to_vec(), SectionKind::Data);
let address_size = self
.object
.architecture()
.address_size()
.expect("address_size must be known")
.bytes();
self.object.append_section_data(
init_array_section,
&std::iter::repeat(0)
.take(address_size.into())
.collect::<Vec<u8>>(),
8,
);
self.object
.add_relocation(
init_array_section,
object::write::Relocation {
offset: 0,
size: address_size * 8,
kind: RelocationKind::Absolute,
encoding: RelocationEncoding::Generic,
symbol,
addend: 0,
},
)
.unwrap();
}
}
pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
let triple = crate::build_isa(sess).triple().clone();
@ -175,10 +126,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
architecture => sess.fatal(&format!(
"target architecture {:?} is unsupported",
architecture,
)),
architecture => {
sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
}
};
let endian = match triple.endianness().unwrap() {
target_lexicon::Endianness::Little => object::Endianness::Little,

269
compiler/rustc_codegen_cranelift/src/base.rs
View File

@ -8,7 +8,7 @@ use rustc_target::abi::call::FnAbi;
use crate::prelude::*;
pub(crate) fn codegen_fn<'tcx>(
cx: &mut crate::CodegenCx<'tcx, impl Module>,
cx: &mut crate::CodegenCx<'_, 'tcx>,
instance: Instance<'tcx>,
linkage: Linkage,
) {
@ -38,9 +38,8 @@ pub(crate) fn codegen_fn<'tcx>(
// Predefine blocks
let start_block = bcx.create_block();
let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len())
.map(|_| bcx.create_block())
.collect();
let block_map: IndexVec<BasicBlock, Block> =
(0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
// Make FunctionCx
let pointer_type = cx.module.target_config().pointer_type();
@ -68,22 +67,23 @@ pub(crate) fn codegen_fn<'tcx>(
inline_asm_index: 0,
};
let arg_uninhabited = fx.mir.args_iter().any(|arg| {
fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty))
.abi
.is_uninhabited()
});
let arg_uninhabited = fx
.mir
.args_iter()
.any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
if arg_uninhabited {
fx.bcx
.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
if !crate::constant::check_constants(&mut fx) {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
} else if arg_uninhabited {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument");
} else {
tcx.sess.time("codegen clif ir", || {
tcx.sess.time("codegen prelude", || {
crate::abi::codegen_fn_prelude(&mut fx, start_block)
});
tcx.sess
.time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
codegen_fn_content(&mut fx);
});
}
@ -131,11 +131,7 @@ pub(crate) fn codegen_fn<'tcx>(
let module = &mut cx.module;
tcx.sess.time("define function", || {
module
.define_function(
func_id,
context,
&mut cranelift_codegen::binemit::NullTrapSink {},
)
.define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
.unwrap()
});
@ -149,14 +145,12 @@ pub(crate) fn codegen_fn<'tcx>(
&clif_comments,
);
if let Some(mach_compile_result) = &context.mach_compile_result {
if let Some(disasm) = &mach_compile_result.disasm {
crate::pretty_clif::write_ir_file(
tcx,
&format!("{}.vcode", tcx.symbol_name(instance).name),
|file| file.write_all(disasm.as_bytes()),
)
}
if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
crate::pretty_clif::write_ir_file(
tcx,
&format!("{}.vcode", tcx.symbol_name(instance).name),
|file| file.write_all(disasm.as_bytes()),
)
}
// Define debuginfo for function
@ -199,16 +193,13 @@ pub(crate) fn verify_func(
Some(Box::new(writer)),
err,
);
tcx.sess
.fatal(&format!("cranelift verify error:\n{}", pretty_error));
tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error));
}
}
});
}
fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
crate::constant::check_constants(fx);
fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
let block = fx.get_block(bb);
fx.bcx.switch_to_block(block);
@ -231,11 +222,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
#[cfg(debug_assertions)]
{
let mut terminator_head = "\n".to_string();
bb_data
.terminator()
.kind
.fmt_head(&mut terminator_head)
.unwrap();
bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
let inst = fx.bcx.func.layout.last_inst(block).unwrap();
fx.add_comment(inst, terminator_head);
}
@ -267,13 +254,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
TerminatorKind::Return => {
crate::abi::codegen_return(fx);
}
TerminatorKind::Assert {
cond,
expected,
msg,
target,
cleanup: _,
} => {
TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
if !fx.tcx.sess.overflow_checks() {
if let mir::AssertKind::OverflowNeg(_) = *msg {
let target = fx.get_block(*target);
@ -319,11 +300,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
}
}
TerminatorKind::SwitchInt {
discr,
switch_ty,
targets,
} => {
TerminatorKind::SwitchInt { discr, switch_ty, targets } => {
let discr = codegen_operand(fx, discr).load_scalar(fx);
let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
@ -433,11 +410,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
| TerminatorKind::GeneratorDrop => {
bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
}
TerminatorKind::Drop {
place,
target,
unwind: _,
} => {
TerminatorKind::Drop { place, target, unwind: _ } => {
let drop_place = codegen_place(fx, *place);
crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
@ -452,7 +425,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
}
fn codegen_stmt<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
#[allow(unused_variables)] cur_block: Block,
stmt: &Statement<'tcx>,
) {
@ -470,10 +443,7 @@ fn codegen_stmt<'tcx>(
}
match &stmt.kind {
StatementKind::SetDiscriminant {
place,
variant_index,
} => {
StatementKind::SetDiscriminant { place, variant_index } => {
let place = codegen_place(fx, **place);
crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
}
@ -594,19 +564,11 @@ fn codegen_stmt<'tcx>(
let from_ty = operand.layout().ty;
let to_ty = fx.monomorphize(to_ty);
fn is_fat_ptr<'tcx>(
fx: &FunctionCx<'_, 'tcx, impl Module>,
ty: Ty<'tcx>,
) -> bool {
fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
ty.builtin_deref(true)
.map(
|ty::TypeAndMut {
ty: pointee_ty,
mutbl: _,
}| {
has_ptr_meta(fx.tcx, pointee_ty)
},
)
.map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| {
has_ptr_meta(fx.tcx, pointee_ty)
})
.unwrap_or(false)
}
@ -626,50 +588,22 @@ fn codegen_stmt<'tcx>(
ty::Uint(_) | ty::Int(_) => {}
_ => unreachable!("cast adt {} -> {}", from_ty, to_ty),
}
let to_clif_ty = fx.clif_type(to_ty).unwrap();
use rustc_target::abi::{Int, TagEncoding, Variants};
let discriminant = crate::discriminant::codegen_get_discriminant(
fx,
operand,
fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)),
)
.load_scalar(fx);
match operand.layout().variants {
Variants::Single { index } => {
let discr = operand
.layout()
.ty
.discriminant_for_variant(fx.tcx, index)
.unwrap();
let discr = if discr.ty.is_signed() {
fx.layout_of(discr.ty).size.sign_extend(discr.val)
} else {
discr.val
};
let discr = discr.into();
let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr);
lval.write_cvalue(fx, discr);
}
Variants::Multiple {
ref tag,
tag_field,
tag_encoding: TagEncoding::Direct,
variants: _,
} => {
let cast_to = fx.clif_type(dest_layout.ty).unwrap();
// Read the tag/niche-encoded discriminant from memory.
let encoded_discr =
operand.value_field(fx, mir::Field::new(tag_field));
let encoded_discr = encoded_discr.load_scalar(fx);
// Decode the discriminant (specifically if it's niche-encoded).
let signed = match tag.value {
Int(_, signed) => signed,
_ => false,
};
let val = clif_intcast(fx, encoded_discr, cast_to, signed);
let val = CValue::by_val(val, dest_layout);
lval.write_cvalue(fx, val);
}
Variants::Multiple { .. } => unreachable!(),
}
let res = crate::cast::clif_intcast(
fx,
discriminant,
to_clif_ty,
to_ty.is_signed(),
);
lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
} else {
let to_clif_ty = fx.clif_type(to_ty).unwrap();
let from = operand.load_scalar(fx);
@ -730,8 +664,7 @@ fn codegen_stmt<'tcx>(
// FIXME use emit_small_memset where possible
let addr = lval.to_ptr().get_addr(fx);
let val = operand.load_scalar(fx);
fx.bcx
.call_memset(fx.cx.module.target_config(), addr, val, times);
fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times);
} else {
let loop_block = fx.bcx.create_block();
let loop_block2 = fx.bcx.create_block();
@ -766,25 +699,19 @@ fn codegen_stmt<'tcx>(
let content_ty = fx.monomorphize(content_ty);
let layout = fx.layout_of(content_ty);
let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64);
let llalign = fx
.bcx
.ins()
.iconst(usize_type, layout.align.abi.bytes() as i64);
let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64);
let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
// Allocate space:
let def_id = match fx
.tcx
.lang_items()
.require(rustc_hir::LangItem::ExchangeMalloc)
{
Ok(id) => id,
Err(s) => {
fx.tcx
.sess
.fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
}
};
let def_id =
match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) {
Ok(id) => id,
Err(s) => {
fx.tcx
.sess
.fatal(&format!("allocation of `{}` {}", box_layout.ty, s));
}
};
let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
let func_ref = fx.get_function_ref(instance);
let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]);
@ -792,10 +719,11 @@ fn codegen_stmt<'tcx>(
lval.write_cvalue(fx, CValue::by_val(ptr, box_layout));
}
Rvalue::NullaryOp(NullOp::SizeOf, ty) => {
assert!(lval
.layout()
.ty
.is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()));
assert!(
lval.layout()
.ty
.is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())
);
let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes();
let val =
CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into());
@ -823,11 +751,7 @@ fn codegen_stmt<'tcx>(
StatementKind::LlvmInlineAsm(asm) => {
use rustc_span::symbol::Symbol;
let LlvmInlineAsm {
asm,
outputs,
inputs,
} = &**asm;
let LlvmInlineAsm { asm, outputs, inputs } = &**asm;
let rustc_hir::LlvmInlineAsmInner {
asm: asm_code, // Name
outputs: output_names, // Vec<LlvmInlineAsmOutput>
@ -843,15 +767,9 @@ fn codegen_stmt<'tcx>(
// Black box
}
"mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => {
assert_eq!(
input_names,
&[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]
);
assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
assert_eq!(output_names.len(), 4);
for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"])
.iter()
.enumerate()
{
for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
assert_eq!(&output_names[i].constraint.as_str(), c);
assert!(!output_names[i].is_rw);
assert!(!output_names[i].is_indirect);
@ -897,12 +815,7 @@ fn codegen_stmt<'tcx>(
crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
}
// ___chkstk, ___chkstk_ms and __alloca are only used on Windows
_ if fx
.tcx
.symbol_name(fx.instance)
.name
.starts_with("___chkstk") =>
{
_ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
}
_ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
@ -922,27 +835,21 @@ fn codegen_stmt<'tcx>(
}
}
fn codegen_array_len<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
place: CPlace<'tcx>,
) -> Value {
fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
match *place.layout().ty.kind() {
ty::Array(_elem_ty, len) => {
let len = fx
.monomorphize(len)
.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
fx.bcx.ins().iconst(fx.pointer_type, len)
}
ty::Slice(_elem_ty) => place
.to_ptr_maybe_unsized()
.1
.expect("Length metadata for slice place"),
ty::Slice(_elem_ty) => {
place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
}
_ => bug!("Rvalue::Len({:?})", place),
}
}
pub(crate) fn codegen_place<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
place: Place<'tcx>,
) -> CPlace<'tcx> {
let mut cplace = fx.get_local_place(place.local);
@ -959,11 +866,7 @@ pub(crate) fn codegen_place<'tcx>(
let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
cplace = cplace.place_index(fx, index);
}
PlaceElem::ConstantIndex {
offset,
min_length: _,
from_end,
} => {
PlaceElem::ConstantIndex { offset, min_length: _, from_end } => {
let offset: u64 = offset;
let index = if !from_end {
fx.bcx.ins().iconst(fx.pointer_type, offset as i64)
@ -1014,7 +917,7 @@ pub(crate) fn codegen_place<'tcx>(
}
pub(crate) fn codegen_operand<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
operand: &Operand<'tcx>,
) -> CValue<'tcx> {
match operand {
@ -1026,34 +929,24 @@ pub(crate) fn codegen_operand<'tcx>(
}
}
pub(crate) fn codegen_panic<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
msg_str: &str,
span: Span,
) {
pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
let location = fx.get_caller_location(span).load_scalar(fx);
let msg_ptr = fx.anonymous_str("assert", msg_str);
let msg_len = fx
.bcx
.ins()
.iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let args = [msg_ptr, msg_len, location];
codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span);
}
pub(crate) fn codegen_panic_inner<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
lang_item: rustc_hir::LangItem,
args: &[Value],
span: Span,
) {
let def_id = fx
.tcx
.lang_items()
.require(lang_item)
.unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
let def_id =
fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
let symbol_name = fx.tcx.symbol_name(instance).name;

8
compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
View File

@ -27,13 +27,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
config.opts.cg.panic = Some(PanicStrategy::Abort);
config.opts.debugging_opts.panic_abort_tests = true;
config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| {
std::env::current_exe()
.unwrap()
.parent()
.unwrap()
.parent()
.unwrap()
.to_owned()
std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()
}));
}
}

11
compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
View File

@ -46,15 +46,8 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
config.opts.cg.panic = Some(PanicStrategy::Abort);
config.opts.debugging_opts.panic_abort_tests = true;
config.opts.maybe_sysroot = Some(
std::env::current_exe()
.unwrap()
.parent()
.unwrap()
.parent()
.unwrap()
.to_owned(),
);
config.opts.maybe_sysroot =
Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned());
}
}

34
compiler/rustc_codegen_cranelift/src/cast.rs
View File

@ -3,7 +3,7 @@
use crate::prelude::*;
pub(crate) fn clif_intcast(
fx: &mut FunctionCx<'_, '_, impl Module>,
fx: &mut FunctionCx<'_, '_, '_>,
val: Value,
to: Type,
signed: bool,
@ -40,18 +40,14 @@ pub(crate) fn clif_intcast(
// reduce
(types::I128, _) => {
let (lsb, _msb) = fx.bcx.ins().isplit(val);
if to == types::I64 {
lsb
} else {
fx.bcx.ins().ireduce(to, lsb)
}
if to == types::I64 { lsb } else { fx.bcx.ins().ireduce(to, lsb) }
}
(_, _) => fx.bcx.ins().ireduce(to, val),
}
}
pub(crate) fn clif_int_or_float_cast(
fx: &mut FunctionCx<'_, '_, impl Module>,
fx: &mut FunctionCx<'_, '_, '_>,
from: Value,
from_signed: bool,
to_ty: Type,
@ -87,11 +83,7 @@ pub(crate) fn clif_int_or_float_cast(
},
);
let from_rust_ty = if from_signed {
fx.tcx.types.i128
} else {
fx.tcx.types.u128
};
let from_rust_ty = if from_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
let to_rust_ty = match to_ty {
types::F32 => fx.tcx.types.f32,
@ -100,11 +92,7 @@ pub(crate) fn clif_int_or_float_cast(
};
return fx
.easy_call(
&name,
&[CValue::by_val(from, fx.layout_of(from_rust_ty))],
to_rust_ty,
)
.easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
.load_scalar(fx);
}
@ -138,18 +126,10 @@ pub(crate) fn clif_int_or_float_cast(
_ => unreachable!(),
};
let to_rust_ty = if to_signed {
fx.tcx.types.i128
} else {
fx.tcx.types.u128
};
let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
return fx
.easy_call(
&name,
&[CValue::by_val(from, fx.layout_of(from_rust_ty))],
to_rust_ty,
)
.easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
.load_scalar(fx);
}

87
compiler/rustc_codegen_cranelift/src/codegen_i128.rs
View File

@ -5,13 +5,17 @@ use cranelift_codegen::ir::ArgumentPurpose;
use crate::prelude::*;
pub(crate) fn maybe_codegen<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
checked: bool,
lhs: CValue<'tcx>,
rhs: CValue<'tcx>,
) -> Option<CValue<'tcx>> {
if lhs.layout().ty != fx.tcx.types.u128 && lhs.layout().ty != fx.tcx.types.i128 {
if lhs.layout().ty != fx.tcx.types.u128
&& lhs.layout().ty != fx.tcx.types.i128
&& rhs.layout().ty != fx.tcx.types.u128
&& rhs.layout().ty != fx.tcx.types.i128
{
return None;
}
@ -27,11 +31,7 @@ pub(crate) fn maybe_codegen<'tcx>(
}
BinOp::Add | BinOp::Sub if !checked => None,
BinOp::Mul if !checked => {
let val_ty = if is_signed {
fx.tcx.types.i128
} else {
fx.tcx.types.u128
};
let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
}
BinOp::Add | BinOp::Sub | BinOp::Mul => {
@ -43,11 +43,7 @@ pub(crate) fn maybe_codegen<'tcx>(
AbiParam::new(types::I128),
AbiParam::new(types::I128),
];
let args = [
out_place.to_ptr().get_addr(fx),
lhs.load_scalar(fx),
rhs.load_scalar(fx),
];
let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
let name = match (bin_op, is_signed) {
(BinOp::Add, false) => "__rust_u128_addo",
(BinOp::Add, true) => "__rust_i128_addo",
@ -97,70 +93,23 @@ pub(crate) fn maybe_codegen<'tcx>(
None
};
// Optimize `val >> 64`, because compiler_builtins uses it to deconstruct an 128bit
// integer into its lsb and msb.
// https://github.com/rust-lang-nursery/compiler-builtins/blob/79a6a1603d5672cbb9187ff41ff4d9b5048ac1cb/src/int/mod.rs#L217
if resolve_value_imm(fx.bcx.func, rhs_val) == Some(64) {
let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs_val);
let all_zeros = fx.bcx.ins().iconst(types::I64, 0);
let val = match (bin_op, is_signed) {
(BinOp::Shr, false) => {
let val = fx.bcx.ins().iconcat(lhs_msb, all_zeros);
Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.u128)))
}
(BinOp::Shr, true) => {
let sign = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, lhs_msb, 0);
let all_ones = fx.bcx.ins().iconst(types::I64, u64::MAX as i64);
let all_sign_bits = fx.bcx.ins().select(sign, all_zeros, all_ones);
let val = fx.bcx.ins().iconcat(lhs_msb, all_sign_bits);
Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.i128)))
}
(BinOp::Shl, _) => {
let val_ty = if is_signed {
fx.tcx.types.i128
} else {
fx.tcx.types.u128
};
let val = fx.bcx.ins().iconcat(all_zeros, lhs_lsb);
Some(CValue::by_val(val, fx.layout_of(val_ty)))
}
_ => None,
};
if let Some(val) = val {
if let Some(is_overflow) = is_overflow {
let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
let val = val.load_scalar(fx);
return Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)));
} else {
return Some(val);
}
}
}
let truncated_rhs = clif_intcast(fx, rhs_val, types::I32, false);
let truncated_rhs = CValue::by_val(truncated_rhs, fx.layout_of(fx.tcx.types.u32));
let val = match (bin_op, is_signed) {
(BinOp::Shl, false) => {
fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
let val = match bin_op {
BinOp::Shl => fx.bcx.ins().ishl(lhs_val, truncated_rhs),
BinOp::Shr => {
if is_signed {
fx.bcx.ins().sshr(lhs_val, truncated_rhs)
} else {
fx.bcx.ins().ushr(lhs_val, truncated_rhs)
}
}
(BinOp::Shl, true) => {
fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
}
(BinOp::Shr, false) => {
fx.easy_call("__lshrti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
}
(BinOp::Shr, true) => {
fx.easy_call("__ashrti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
}
(_, _) => unreachable!(),
_ => unreachable!(),
};
if let Some(is_overflow) = is_overflow {
let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
let val = val.load_scalar(fx);
Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)))
} else {
Some(val)
Some(CValue::by_val(val, lhs.layout()))
}
}
}

111
compiler/rustc_codegen_cranelift/src/common.rs
View File

@ -3,8 +3,6 @@ use rustc_target::abi::call::FnAbi;
use rustc_target::abi::{Integer, Primitive};
use rustc_target::spec::{HasTargetSpec, Target};
use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef};
use crate::prelude::*;
pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@ -56,11 +54,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
FloatTy::F64 => types::F64,
},
ty::FnPtr(_) => pointer_ty(tcx),
ty::RawPtr(TypeAndMut {
ty: pointee_ty,
mutbl: _,
})
| ty::Ref(_, pointee_ty, _) => {
ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
if has_ptr_meta(tcx, pointee_ty) {
return None;
} else {
@ -99,11 +93,7 @@ fn clif_pair_type_from_ty<'tcx>(
}
(a, b)
}
ty::RawPtr(TypeAndMut {
ty: pointee_ty,
mutbl: _,
})
| ty::Ref(_, pointee_ty, _) => {
ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
if has_ptr_meta(tcx, pointee_ty) {
(pointer_ty(tcx), pointer_ty(tcx))
} else {
@ -116,15 +106,8 @@ fn clif_pair_type_from_ty<'tcx>(
/// Is a pointer to this type a fat ptr?
pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
let ptr_ty = tcx.mk_ptr(TypeAndMut {
ty,
mutbl: rustc_hir::Mutability::Not,
});
match &tcx
.layout_of(ParamEnv::reveal_all().and(ptr_ty))
.unwrap()
.abi
{
let ptr_ty = tcx.mk_ptr(TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not });
match &tcx.layout_of(ParamEnv::reveal_all().and(ptr_ty)).unwrap().abi {
Abi::Scalar(_) => false,
Abi::ScalarPair(_, _) => true,
abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi),
@ -132,7 +115,7 @@ pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
}
pub(crate) fn codegen_icmp_imm(
fx: &mut FunctionCx<'_, '_, impl Module>,
fx: &mut FunctionCx<'_, '_, '_>,
intcc: IntCC,
lhs: Value,
rhs: i128,
@ -175,51 +158,6 @@ pub(crate) fn codegen_icmp_imm(
}
}
fn resolve_normal_value_imm(func: &Function, val: Value) -> Option<i64> {
if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
if let InstructionData::UnaryImm {
opcode: Opcode::Iconst,
imm,
} = func.dfg[inst]
{
Some(imm.into())
} else {
None
}
} else {
None
}
}
fn resolve_128bit_value_imm(func: &Function, val: Value) -> Option<u128> {
let (lsb, msb) = if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
if let InstructionData::Binary {
opcode: Opcode::Iconcat,
args: [lsb, msb],
} = func.dfg[inst]
{
(lsb, msb)
} else {
return None;
}
} else {
return None;
};
let lsb = u128::from(resolve_normal_value_imm(func, lsb)? as u64);
let msb = u128::from(resolve_normal_value_imm(func, msb)? as u64);
Some(msb << 64 | lsb)
}
pub(crate) fn resolve_value_imm(func: &Function, val: Value) -> Option<u128> {
if func.dfg.value_type(val) == types::I128 {
resolve_128bit_value_imm(func, val)
} else {
resolve_normal_value_imm(func, val).map(|imm| u128::from(imm as u64))
}
}
pub(crate) fn type_min_max_value(
bcx: &mut FunctionBuilder<'_>,
ty: Type,
@ -288,8 +226,8 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
}
}
pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
pub(crate) cx: &'clif mut crate::CodegenCx<'tcx, M>,
pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>,
pub(crate) tcx: TyCtxt<'tcx>,
pub(crate) pointer_type: Type, // Cached from module
@ -316,7 +254,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
pub(crate) inline_asm_index: u32,
}
impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
impl<'tcx> LayoutOf for FunctionCx<'_, '_, 'tcx> {
type Ty = Ty<'tcx>;
type TyAndLayout = TyAndLayout<'tcx>;
@ -325,31 +263,31 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
}
}
impl<'tcx, M: Module> layout::HasTyCtxt<'tcx> for FunctionCx<'_, 'tcx, M> {
impl<'tcx> layout::HasTyCtxt<'tcx> for FunctionCx<'_, '_, 'tcx> {
fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
self.tcx
}
}
impl<'tcx, M: Module> rustc_target::abi::HasDataLayout for FunctionCx<'_, 'tcx, M> {
impl<'tcx> rustc_target::abi::HasDataLayout for FunctionCx<'_, '_, 'tcx> {
fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
&self.tcx.data_layout
}
}
impl<'tcx, M: Module> layout::HasParamEnv<'tcx> for FunctionCx<'_, 'tcx, M> {
impl<'tcx> layout::HasParamEnv<'tcx> for FunctionCx<'_, '_, 'tcx> {
fn param_env(&self) -> ParamEnv<'tcx> {
ParamEnv::reveal_all()
}
}
impl<'tcx, M: Module> HasTargetSpec for FunctionCx<'_, 'tcx, M> {
impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
fn target_spec(&self) -> &Target {
&self.tcx.sess.target
}
}
impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
impl<'tcx> FunctionCx<'_, '_, 'tcx> {
pub(crate) fn monomorphize<T>(&self, value: T) -> T
where
T: TypeFoldable<'tcx> + Copy,
@ -416,12 +354,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
let msg_id = self
.cx
.module
.declare_data(
&format!("__{}_{:08x}", prefix, msg_hash),
Linkage::Local,
false,
false,
)
.declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false)
.unwrap();
// Ignore DuplicateDefinition error, as the data will be the same
@ -444,15 +377,13 @@ impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> {
fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> {
assert!(!ty.still_further_specializable());
self.0
.layout_of(ParamEnv::reveal_all().and(&ty))
.unwrap_or_else(|e| {
if let layout::LayoutError::SizeOverflow(_) = e {
self.0.sess.fatal(&e.to_string())
} else {
bug!("failed to get layout for `{}`: {}", ty, e)
}
})
self.0.layout_of(ParamEnv::reveal_all().and(&ty)).unwrap_or_else(|e| {
if let layout::LayoutError::SizeOverflow(_) = e {
self.0.sess.fatal(&e.to_string())
} else {
bug!("failed to get layout for `{}`: {}", ty, e)
}
})
}
}

118
compiler/rustc_codegen_cranelift/src/constant.rs
View File

@ -28,7 +28,7 @@ enum TodoItem {
}
impl ConstantCx {
pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut impl Module) {
pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {
//println!("todo {:?}", self.todo);
define_all_allocs(tcx, module, &mut self);
//println!("done {:?}", self.done);
@ -36,21 +36,20 @@ impl ConstantCx {
}
}
pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
let mut all_constants_ok = true;
for constant in &fx.mir.required_consts {
let const_ = fx.monomorphize(constant.literal);
match const_.val {
ConstKind::Value(_) => {}
ConstKind::Unevaluated(def, ref substs, promoted) => {
if let Err(err) =
fx.tcx
.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
{
all_constants_ok = false;
match err {
ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {
fx.tcx
.sess
.span_err(constant.span, "erroneous constant encountered");
fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
}
ErrorHandled::TooGeneric => {
span_bug!(
@ -69,6 +68,7 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
| ConstKind::Error(_) => unreachable!("{:?}", const_),
}
}
all_constants_ok
}
pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
@ -76,11 +76,11 @@ pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
}
pub(crate) fn codegen_tls_ref<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
def_id: DefId,
layout: TyAndLayout<'tcx>,
) -> CValue<'tcx> {
let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("tls {:?}", def_id));
@ -89,11 +89,11 @@ pub(crate) fn codegen_tls_ref<'tcx>(
}
fn codegen_static_ref<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
def_id: DefId,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("{:?}", def_id));
@ -110,7 +110,7 @@ fn codegen_static_ref<'tcx>(
}
pub(crate) fn codegen_constant<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
constant: &Constant<'tcx>,
) -> CValue<'tcx> {
let const_ = fx.monomorphize(constant.literal);
@ -128,20 +128,10 @@ pub(crate) fn codegen_constant<'tcx>(
.to_cvalue(fx);
}
ConstKind::Unevaluated(def, ref substs, promoted) => {
match fx
.tcx
.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
{
match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) {
Ok(const_val) => const_val,
Err(_) => {
fx.tcx
.sess
.span_err(constant.span, "erroneous constant encountered");
return crate::trap::trap_unreachable_ret_value(
fx,
fx.layout_of(const_.ty),
"erroneous constant encountered",
);
span_bug!(constant.span, "erroneous constant not captured by required_consts");
}
}
}
@ -156,7 +146,7 @@ pub(crate) fn codegen_constant<'tcx>(
}
pub(crate) fn codegen_const_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
const_val: ConstValue<'tcx>,
ty: Ty<'tcx>,
) -> CValue<'tcx> {
@ -172,9 +162,7 @@ pub(crate) fn codegen_const_value<'tcx>(
if fx.clif_type(layout.ty).is_none() {
let (size, align) = (layout.size, layout.align.pref);
let mut alloc = Allocation::from_bytes(
std::iter::repeat(0)
.take(size.bytes_usize())
.collect::<Vec<u8>>(),
std::iter::repeat(0).take(size.bytes_usize()).collect::<Vec<u8>>(),
align,
);
let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used
@ -190,11 +178,8 @@ pub(crate) fn codegen_const_value<'tcx>(
let base_addr = match alloc_kind {
Some(GlobalAlloc::Memory(alloc)) => {
fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
let data_id = data_id_for_alloc_id(
&mut fx.cx.module,
ptr.alloc_id,
alloc.mutability,
);
let data_id =
data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
@ -203,15 +188,14 @@ pub(crate) fn codegen_const_value<'tcx>(
}
Some(GlobalAlloc::Function(instance)) => {
let func_id =
crate::abi::import_function(fx.tcx, &mut fx.cx.module, instance);
crate::abi::import_function(fx.tcx, fx.cx.module, instance);
let local_func_id =
fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
}
Some(GlobalAlloc::Static(def_id)) => {
assert!(fx.tcx.is_static(def_id));
let data_id =
data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
@ -221,9 +205,7 @@ pub(crate) fn codegen_const_value<'tcx>(
None => bug!("missing allocation {:?}", ptr.alloc_id),
};
let val = if ptr.offset.bytes() != 0 {
fx.bcx
.ins()
.iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
} else {
base_addr
};
@ -240,22 +222,22 @@ pub(crate) fn codegen_const_value<'tcx>(
let ptr = pointer_for_allocation(fx, data)
.offset_i64(fx, i64::try_from(start).unwrap())
.get_addr(fx);
let len = fx.bcx.ins().iconst(
fx.pointer_type,
i64::try_from(end.checked_sub(start).unwrap()).unwrap(),
);
let len = fx
.bcx
.ins()
.iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
CValue::by_val_pair(ptr, len, layout)
}
}
}
fn pointer_for_allocation<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
alloc: &'tcx Allocation,
) -> crate::pointer::Pointer {
let alloc_id = fx.tcx.create_memory_alloc(alloc);
fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
let data_id = data_id_for_alloc_id(&mut fx.cx.module, alloc_id, alloc.mutability);
let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
@ -265,7 +247,7 @@ fn pointer_for_allocation<'tcx>(
}
fn data_id_for_alloc_id(
module: &mut impl Module,
module: &mut dyn Module,
alloc_id: AllocId,
mutability: rustc_hir::Mutability,
) -> DataId {
@ -281,7 +263,7 @@ fn data_id_for_alloc_id(
fn data_id_for_static(
tcx: TyCtxt<'_>,
module: &mut impl Module,
module: &mut dyn Module,
def_id: DefId,
definition: bool,
) -> DataId {
@ -304,12 +286,7 @@ fn data_id_for_static(
} else {
!ty.is_freeze(tcx.at(DUMMY_SP), ParamEnv::reveal_all())
};
let align = tcx
.layout_of(ParamEnv::reveal_all().and(ty))
.unwrap()
.align
.pref
.bytes();
let align = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().align.pref.bytes();
let attrs = tcx.codegen_fn_attrs(def_id);
@ -332,17 +309,11 @@ fn data_id_for_static(
// zero.
let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name);
let ref_data_id = module
.declare_data(&ref_name, Linkage::Local, false, false)
.unwrap();
let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap();
let mut data_ctx = DataContext::new();
data_ctx.set_align(align);
let data = module.declare_data_in_data(data_id, &mut data_ctx);
data_ctx.define(
std::iter::repeat(0)
.take(pointer_ty(tcx).bytes() as usize)
.collect(),
);
data_ctx.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect());
data_ctx.write_data_addr(0, data, 0);
match module.define_data(ref_data_id, &data_ctx) {
// Every time the static is referenced there will be another definition of this global,
@ -356,7 +327,7 @@ fn data_id_for_static(
}
}
fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut ConstantCx) {
fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) {
while let Some(todo_item) = cx.todo.pop() {
let (data_id, alloc, section_name) = match todo_item {
TodoItem::Alloc(alloc_id) => {
@ -371,10 +342,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
TodoItem::Static(def_id) => {
//println!("static {:?}", def_id);
let section_name = tcx
.codegen_fn_attrs(def_id)
.link_section
.map(|s| s.as_str());
let section_name = tcx.codegen_fn_attrs(def_id).link_section.map(|s| s.as_str());
let alloc = tcx.eval_static_initializer(def_id).unwrap();
@ -396,9 +364,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
data_ctx.set_segment_section("", &*section_name);
}
let bytes = alloc
.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
.to_vec();
let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
data_ctx.define(bytes.into_boxed_slice());
for &(offset, (_tag, reloc)) in alloc.relocations().iter() {
@ -426,10 +392,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
data_id_for_alloc_id(module, reloc, target_alloc.mutability)
}
GlobalAlloc::Static(def_id) => {
if tcx
.codegen_fn_attrs(def_id)
.flags
.contains(CodegenFnAttrFlags::THREAD_LOCAL)
if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
{
tcx.sess.fatal(&format!(
"Allocation {:?} contains reference to TLS value {:?}",
@ -457,14 +420,13 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
}
pub(crate) fn mir_operand_get_const_val<'tcx>(
fx: &FunctionCx<'_, 'tcx, impl Module>,
fx: &FunctionCx<'_, '_, 'tcx>,
operand: &Operand<'tcx>,
) -> Option<&'tcx Const<'tcx>> {
match operand {
Operand::Copy(_) | Operand::Move(_) => None,
Operand::Constant(const_) => Some(
fx.monomorphize(const_.literal)
.eval(fx.tcx, ParamEnv::reveal_all()),
),
Operand::Constant(const_) => {
Some(fx.monomorphize(const_.literal).eval(fx.tcx, ParamEnv::reveal_all()))
}
}
}

14
compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
View File

@ -14,10 +14,7 @@ impl DebugContext<'_> {
let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
let root = self.dwarf.unit.root();
let root = self.dwarf.unit.get_mut(root);
root.set(
gimli::DW_AT_ranges,
AttributeValue::RangeListRef(unit_range_list_id),
);
root.set(gimli::DW_AT_ranges, AttributeValue::RangeListRef(unit_range_list_id));
let mut sections = Sections::new(WriterRelocate::new(self.endian));
self.dwarf.write(&mut sections).unwrap();
@ -66,10 +63,7 @@ pub(super) struct WriterRelocate {
impl WriterRelocate {
pub(super) fn new(endian: RunTimeEndian) -> Self {
WriterRelocate {
relocs: Vec::new(),
writer: EndianVec::new(endian),
}
WriterRelocate { relocs: Vec::new(), writer: EndianVec::new(endian) }
}
/// Perform the collected relocations to be usable for JIT usage.
@ -85,9 +79,7 @@ impl WriterRelocate {
cranelift_module::FuncId::from_u32(sym.try_into().unwrap()),
);
let val = (addr as u64 as i64 + reloc.addend) as u64;
self.writer
.write_udata_at(reloc.offset as usize, val, reloc.size)
.unwrap();
self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap();
}
}
}

75
compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
View File

@ -53,11 +53,7 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
if hash.kind == SourceFileHashAlgorithm::Md5 {
let mut buf = [0u8; MD5_LEN];
buf.copy_from_slice(hash.hash_bytes());
Some(FileInfo {
timestamp: 0,
size: 0,
md5: buf,
})
Some(FileInfo { timestamp: 0, size: 0, md5: buf })
} else {
None
}
@ -112,24 +108,14 @@ impl<'tcx> DebugContext<'tcx> {
let entry = self.dwarf.unit.get_mut(entry_id);
entry.set(
gimli::DW_AT_decl_file,
AttributeValue::FileIndex(Some(file_id)),
);
entry.set(
gimli::DW_AT_decl_line,
AttributeValue::Udata(loc.line as u64),
);
entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
// FIXME: probably omit this
entry.set(
gimli::DW_AT_decl_column,
AttributeValue::Udata(loc.col.to_usize() as u64),
);
entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
}
pub(super) fn create_debug_lines(
&mut self,
isa: &dyn cranelift_codegen::isa::TargetIsa,
symbol: usize,
entry_id: UnitEntryId,
context: &Context,
@ -138,7 +124,6 @@ impl<'tcx> DebugContext<'tcx> {
) -> CodeOffset {
let tcx = self.tcx;
let line_program = &mut self.dwarf.unit.line_program;
let func = &context.func;
let line_strings = &mut self.dwarf.line_strings;
let mut last_span = None;
@ -202,43 +187,22 @@ impl<'tcx> DebugContext<'tcx> {
let mut func_end = 0;
if let Some(ref mcr) = &context.mach_compile_result {
for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
line_program.row().address_offset = u64::from(start);
if !loc.is_default() {
let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
create_row_for_span(line_program, source_info.span);
} else {
create_row_for_span(line_program, function_span);
}
func_end = end;
let mcr = context.mach_compile_result.as_ref().unwrap();
for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
line_program.row().address_offset = u64::from(start);
if !loc.is_default() {
let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
create_row_for_span(line_program, source_info.span);
} else {
create_row_for_span(line_program, function_span);
}
line_program.end_sequence(u64::from(func_end));
func_end = mcr.buffer.total_size();
} else {
let encinfo = isa.encoding_info();
let mut blocks = func.layout.blocks().collect::<Vec<_>>();
blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase
for block in blocks {
for (offset, inst, size) in func.inst_offsets(block, &encinfo) {
let srcloc = func.srclocs[inst];
line_program.row().address_offset = u64::from(offset);
if !srcloc.is_default() {
let source_info =
*source_info_set.get_index(srcloc.bits() as usize).unwrap();
create_row_for_span(line_program, source_info.span);
} else {
create_row_for_span(line_program, function_span);
}
func_end = offset + size;
}
}
line_program.end_sequence(u64::from(func_end));
func_end = end;
}
line_program.end_sequence(u64::from(func_end));
let func_end = mcr.buffer.total_size();
assert_ne!(func_end, 0);
let entry = self.dwarf.unit.get_mut(entry_id);
@ -246,10 +210,7 @@ impl<'tcx> DebugContext<'tcx> {
gimli::DW_AT_low_pc,
AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
);
entry.set(
gimli::DW_AT_high_pc,
AttributeValue::Udata(u64::from(func_end)),
);
entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
self.emit_location(entry_id, function_span);

157
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View File

@ -9,7 +9,7 @@ use crate::prelude::*;
use rustc_index::vec::IndexVec;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::{StackSlots, ValueLabel, ValueLoc};
use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc};
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::ValueLocRange;
@ -39,7 +39,6 @@ pub(crate) struct DebugContext<'tcx> {
dwarf: DwarfUnit,
unit_range_list: RangeList,
clif_types: FxHashMap<Type, UnitEntryId>,
types: FxHashMap<Ty<'tcx>, UnitEntryId>,
}
@ -91,20 +90,11 @@ impl<'tcx> DebugContext<'tcx> {
let root = dwarf.unit.root();
let root = dwarf.unit.get_mut(root);
root.set(
gimli::DW_AT_producer,
AttributeValue::StringRef(dwarf.strings.add(producer)),
);
root.set(
gimli::DW_AT_language,
AttributeValue::Language(gimli::DW_LANG_Rust),
);
root.set(gimli::DW_AT_producer, AttributeValue::StringRef(dwarf.strings.add(producer)));
root.set(gimli::DW_AT_language, AttributeValue::Language(gimli::DW_LANG_Rust));
root.set(gimli::DW_AT_name, AttributeValue::StringRef(name));
root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir));
root.set(
gimli::DW_AT_low_pc,
AttributeValue::Address(Address::Constant(0)),
);
root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
}
DebugContext {
@ -115,48 +105,10 @@ impl<'tcx> DebugContext<'tcx> {
dwarf,
unit_range_list: RangeList(Vec::new()),
clif_types: FxHashMap::default(),
types: FxHashMap::default(),
}
}
fn dwarf_ty_for_clif_ty(&mut self, ty: Type) -> UnitEntryId {
if let Some(type_id) = self.clif_types.get(&ty) {
return *type_id;
}
let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
let primitive = |dwarf: &mut DwarfUnit, ate| {
let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
let type_entry = dwarf.unit.get_mut(type_id);
type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
type_id
};
let type_id = if ty.is_bool() {
primitive(&mut self.dwarf, gimli::DW_ATE_boolean)
} else if ty.is_int() {
primitive(&mut self.dwarf, gimli::DW_ATE_address)
} else if ty.is_float() {
primitive(&mut self.dwarf, gimli::DW_ATE_float)
} else {
new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type)
};
let type_entry = self.dwarf.unit.get_mut(type_id);
type_entry.set(
gimli::DW_AT_name,
AttributeValue::String(format!("{}", ty).replace('i', "u").into_bytes()),
);
type_entry.set(
gimli::DW_AT_byte_size,
AttributeValue::Udata(u64::from(ty.bytes())),
);
type_id
}
fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
if let Some(type_id) = self.types.get(ty) {
return *type_id;
@ -181,10 +133,7 @@ impl<'tcx> DebugContext<'tcx> {
ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
ty::Ref(_, pointee_ty, _mutbl)
| ty::RawPtr(ty::TypeAndMut {
ty: pointee_ty,
mutbl: _mutbl,
}) => {
| ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
// Ensure that type is inserted before recursing to avoid duplicates
@ -211,10 +160,7 @@ impl<'tcx> DebugContext<'tcx> {
let field_offset = layout.fields.offset(field_idx);
let field_layout = layout
.field(
&layout::LayoutCx {
tcx: self.tcx,
param_env: ParamEnv::reveal_all(),
},
&layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
field_idx,
)
.unwrap();
@ -243,10 +189,7 @@ impl<'tcx> DebugContext<'tcx> {
let type_entry = self.dwarf.unit.get_mut(type_id);
type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
type_entry.set(
gimli::DW_AT_byte_size,
AttributeValue::Udata(layout.size.bytes()),
);
type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
self.types.insert(ty, type_id);
@ -286,23 +229,15 @@ impl<'tcx> DebugContext<'tcx> {
let name_id = self.dwarf.strings.add(name);
// Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped.
entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
entry.set(
gimli::DW_AT_linkage_name,
AttributeValue::StringRef(name_id),
);
entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
let end =
self.create_debug_lines(isa, symbol, entry_id, context, mir.span, source_info_set);
let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
self.unit_range_list.0.push(Range::StartLength {
begin: Address::Symbol { symbol, addend: 0 },
length: u64::from(end),
});
if isa.get_mach_backend().is_some() {
return; // Not yet implemented for the AArch64 backend.
}
let func_entry = self.dwarf.unit.get_mut(entry_id);
// Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
func_entry.set(
@ -312,51 +247,6 @@ impl<'tcx> DebugContext<'tcx> {
// Using Udata for DW_AT_high_pc requires at least DWARF4
func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
// FIXME Remove once actual debuginfo for locals works.
for (i, (param, &val)) in context
.func
.signature
.params
.iter()
.zip(
context
.func
.dfg
.block_params(context.func.layout.entry_block().unwrap()),
)
.enumerate()
{
use cranelift_codegen::ir::ArgumentPurpose;
let base_name = match param.purpose {
ArgumentPurpose::Normal => "arg",
ArgumentPurpose::StructArgument(_) => "struct_arg",
ArgumentPurpose::StructReturn => "sret",
ArgumentPurpose::Link
| ArgumentPurpose::FramePointer
| ArgumentPurpose::CalleeSaved => continue,
ArgumentPurpose::VMContext
| ArgumentPurpose::SignatureId
| ArgumentPurpose::CallerTLS
| ArgumentPurpose::CalleeTLS
| ArgumentPurpose::StackLimit => unreachable!(),
};
let name = format!("{}{}", base_name, i);
let dw_ty = self.dwarf_ty_for_clif_ty(param.value_type);
let loc =
translate_loc(isa, context.func.locations[val], &context.func.stack_slots).unwrap();
let arg_id = self
.dwarf
.unit
.add(entry_id, gimli::DW_TAG_formal_parameter);
let var_entry = self.dwarf.unit.get_mut(arg_id);
var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
var_entry.set(gimli::DW_AT_location, AttributeValue::Exprloc(loc));
}
// FIXME make it more reliable and implement scopes before re-enabling this.
if false {
let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap();
@ -376,10 +266,7 @@ impl<'tcx> DebugContext<'tcx> {
context,
&local_map,
&value_labels_ranges,
Place {
local,
projection: ty::List::empty(),
},
Place { local, projection: ty::List::empty() },
);
let var_entry = self.dwarf.unit.get_mut(var_id);
@ -417,10 +304,7 @@ fn place_location<'tcx>(
symbol,
addend: i64::from(value_loc_range.start),
},
end: Address::Symbol {
symbol,
addend: i64::from(value_loc_range.end),
},
end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
data: translate_loc(
isa,
value_loc_range.loc,
@ -463,17 +347,17 @@ fn place_location<'tcx>(
// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
fn translate_loc(
isa: &dyn TargetIsa,
loc: ValueLoc,
loc: LabelValueLoc,
stack_slots: &StackSlots,
) -> Option<Expression> {
match loc {
ValueLoc::Reg(reg) => {
LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
let machine_reg = isa.map_dwarf_register(reg).unwrap();
let mut expr = Expression::new();
expr.op_reg(gimli::Register(machine_reg));
Some(expr)
}
ValueLoc::Stack(ss) => {
LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
if let Some(ss_offset) = stack_slots[ss].offset {
let mut expr = Expression::new();
expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16);
@ -482,6 +366,17 @@ fn translate_loc(
None
}
}
_ => None,
LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => unreachable!(),
LabelValueLoc::Reg(reg) => {
let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
let mut expr = Expression::new();
expr.op_reg(gimli::Register(machine_reg));
Some(expr)
}
LabelValueLoc::SPOffset(offset) => {
let mut expr = Expression::new();
expr.op_breg(X86_64::RSP, offset);
Some(expr)
}
}
}

27
compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
View File

@ -28,11 +28,7 @@ impl<'tcx> UnwindContext<'tcx> {
None
};
UnwindContext {
tcx,
frame_table,
cie_id,
}
UnwindContext { tcx, frame_table, cie_id }
}
pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
@ -46,10 +42,8 @@ impl<'tcx> UnwindContext<'tcx> {
UnwindInfo::SystemV(unwind_info) => {
self.frame_table.add_fde(
self.cie_id.unwrap(),
unwind_info.to_fde(Address::Symbol {
symbol: func_id.as_u32() as usize,
addend: 0,
}),
unwind_info
.to_fde(Address::Symbol { symbol: func_id.as_u32() as usize, addend: 0 }),
);
}
UnwindInfo::WindowsX64(_) => {
@ -60,9 +54,8 @@ impl<'tcx> UnwindContext<'tcx> {
}
pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) {
let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
self.tcx,
)));
let mut eh_frame =
EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
if !eh_frame.0.writer.slice().is_empty() {
@ -82,9 +75,8 @@ impl<'tcx> UnwindContext<'tcx> {
self,
jit_module: &cranelift_jit::JITModule,
) -> Option<UnwindRegistry> {
let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
self.tcx,
)));
let mut eh_frame =
EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
if eh_frame.0.writer.slice().is_empty() {
@ -130,10 +122,7 @@ impl<'tcx> UnwindContext<'tcx> {
registrations.push(ptr as usize);
}
Some(UnwindRegistry {
_frame_table: eh_frame,
registrations,
})
Some(UnwindRegistry { _frame_table: eh_frame, registrations })
}
}

45
compiler/rustc_codegen_cranelift/src/discriminant.rs
View File

@ -7,7 +7,7 @@ use rustc_target::abi::{Int, TagEncoding, Variants};
use crate::prelude::*;
pub(crate) fn codegen_set_discriminant<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
place: CPlace<'tcx>,
variant_index: VariantIdx,
) {
@ -26,11 +26,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
variants: _,
} => {
let ptr = place.place_field(fx, mir::Field::new(tag_field));
let to = layout
.ty
.discriminant_for_variant(fx.tcx, variant_index)
.unwrap()
.val;
let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
let to = if ptr.layout().abi.is_signed() {
ty::ScalarInt::try_from_int(
ptr.layout().size.sign_extend(to) as i128,
@ -46,12 +42,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
Variants::Multiple {
tag: _,
tag_field,
tag_encoding:
TagEncoding::Niche {
dataful_variant,
ref niche_variants,
niche_start,
},
tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
variants: _,
} => {
if variant_index != dataful_variant {
@ -70,7 +61,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
}
pub(crate) fn codegen_get_discriminant<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
value: CValue<'tcx>,
dest_layout: TyAndLayout<'tcx>,
) -> CValue<'tcx> {
@ -101,12 +92,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
};
return CValue::const_val(fx, dest_layout, discr_val);
}
Variants::Multiple {
tag,
tag_field,
tag_encoding,
variants: _,
} => (tag, *tag_field, tag_encoding),
Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
(tag, *tag_field, tag_encoding)
}
};
let cast_to = fx.clif_type(dest_layout.ty).unwrap();
@ -125,11 +113,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
let val = clif_intcast(fx, tag, cast_to, signed);
CValue::by_val(val, dest_layout)
}
TagEncoding::Niche {
dataful_variant,
ref niche_variants,
niche_start,
} => {
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
// whether the result is in range for the niche variants.
@ -146,9 +130,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
tag
} else {
// FIXME handle niche_start > i64::MAX
fx.bcx
.ins()
.iadd_imm(tag, -i64::try_from(niche_start).unwrap())
fx.bcx.ins().iadd_imm(tag, -i64::try_from(niche_start).unwrap())
};
let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
let is_niche = {
@ -176,15 +158,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
} else {
clif_intcast(fx, relative_discr, cast_to, false)
};
fx.bcx
.ins()
.iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
};
let dataful_variant = fx
.bcx
.ins()
.iconst(cast_to, i64::from(dataful_variant.as_u32()));
let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
CValue::by_val(discr, dest_layout)
}

114
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View File

@ -12,11 +12,9 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
use rustc_session::cgu_reuse_tracker::CguReuse;
use rustc_session::config::{DebugInfo, OutputType};
use cranelift_object::{ObjectModule, ObjectProduct};
use cranelift_object::ObjectModule;
use crate::prelude::*;
use crate::backend::AddConstructor;
use crate::{prelude::*, BackendConfig};
fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
let module = crate::backend::make_module(tcx.sess, name);
@ -39,7 +37,6 @@ fn emit_module(
module: ObjectModule,
debug: Option<DebugContext<'_>>,
unwind_context: UnwindContext<'_>,
map_product: impl FnOnce(ObjectProduct) -> ObjectProduct,
) -> ModuleCodegenResult {
let mut product = module.finish();
@ -49,15 +46,10 @@ fn emit_module(
unwind_context.emit(&mut product);
let product = map_product(product);
let tmp_file = tcx
.output_filenames(LOCAL_CRATE)
.temp_path(OutputType::Object, Some(&name));
let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name));
let obj = product.object.write().unwrap();
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.sess
.fatal(&format!("error writing object file: {}", err));
tcx.sess.fatal(&format!("error writing object file: {}", err));
}
let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
@ -71,13 +63,7 @@ fn emit_module(
};
ModuleCodegenResult(
CompiledModule {
name,
kind,
object: Some(tmp_file),
dwarf_object: None,
bytecode: None,
},
CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
work_product,
)
}
@ -117,49 +103,27 @@ fn reuse_workproduct_for_cgu(
}
}
fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodegenResult {
fn module_codegen(
tcx: TyCtxt<'_>,
(backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
) -> ModuleCodegenResult {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut module = new_module(tcx, cgu_name.as_str().to_string());
// Initialize the global atomic mutex using a constructor for proc-macros.
// FIXME implement atomic instructions in Cranelift.
let mut init_atomics_mutex_from_constructor = None;
if tcx
.sess
.crate_types()
.contains(&rustc_session::config::CrateType::ProcMacro)
{
if mono_items.iter().any(|(mono_item, _)| match mono_item {
rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx
.symbol_name(Instance::mono(tcx, *def_id))
.name
.contains("__rustc_proc_macro_decls_"),
_ => false,
}) {
init_atomics_mutex_from_constructor =
Some(crate::atomic_shim::init_global_lock_constructor(
&mut module,
&format!("{}_init_atomics_mutex", cgu_name.as_str()),
));
}
}
let mut cx = crate::CodegenCx::new(
tcx,
module,
backend_config,
&mut module,
tcx.sess.opts.debuginfo != DebugInfo::None,
true,
);
super::predefine_mono_items(&mut cx, &mono_items);
for (mono_item, (linkage, visibility)) in mono_items {
let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
match mono_item {
MonoItem::Fn(inst) => {
cx.tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, inst, linkage)
});
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@ -175,9 +139,9 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
}
}
}
let (mut module, global_asm, debug, mut unwind_context) =
let (global_asm, debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false);
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
let codegen_result = emit_module(
tcx,
@ -186,13 +150,6 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
module,
debug,
unwind_context,
|mut product| {
if let Some(func_id) = init_atomics_mutex_from_constructor {
product.add_constructor(func_id);
}
product
},
);
codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
@ -202,6 +159,7 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
pub(super) fn run_aot(
tcx: TyCtxt<'_>,
backend_config: BackendConfig,
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
@ -225,9 +183,7 @@ pub(super) fn run_aot(
cgus.iter()
.map(|cgu| {
let cgu_reuse = determine_cgu_reuse(tcx, cgu);
tcx.sess
.cgu_reuse_tracker
.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
match cgu_reuse {
_ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
@ -242,7 +198,7 @@ pub(super) fn run_aot(
let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
dep_node,
tcx,
cgu.name(),
(backend_config, cgu.name()),
module_codegen,
rustc_middle::dep_graph::hash_result,
);
@ -271,7 +227,6 @@ pub(super) fn run_aot(
allocator_module,
None,
allocator_unwind_context,
|product| product,
);
if let Some((id, product)) = work_product {
work_products.insert(id, product);
@ -301,8 +256,7 @@ pub(super) fn run_aot(
});
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.sess
.fatal(&format!("error writing metadata object file: {}", err));
tcx.sess.fatal(&format!("error writing metadata object file: {}", err));
}
(metadata_cgu_name, tmp_file)
@ -356,8 +310,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
"asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
);
} else {
tcx.sess
.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
}
}
@ -367,19 +320,12 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
// Remove all LLVM style comments
let global_asm = global_asm
.lines()
.map(|line| {
if let Some(index) = line.find("//") {
&line[0..index]
} else {
line
}
})
.map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
.collect::<Vec<_>>()
.join("\n");
let output_object_file = tcx
.output_filenames(LOCAL_CRATE)
.temp_path(OutputType::Object, Some(cgu_name));
let output_object_file =
tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name));
// Assemble `global_asm`
let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
@ -389,16 +335,10 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
.stdin(Stdio::piped())
.spawn()
.expect("Failed to spawn `as`.");
child
.stdin
.take()
.unwrap()
.write_all(global_asm.as_bytes())
.unwrap();
child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
let status = child.wait().expect("Failed to wait for `as`.");
if !status.success() {
tcx.sess
.fatal(&format!("Failed to assemble `{}`", global_asm));
tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
}
// Link the global asm and main object file together
@ -442,11 +382,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
}
let work_product_id = &cgu.work_product_id();
if tcx
.dep_graph
.previous_work_product(work_product_id)
.is_none()
{
if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
// We don't have anything cached for this CGU. This can happen
// if the CGU did not exist in the previous session.
return CguReuse::No;

115
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View File

@ -10,43 +10,24 @@ use rustc_middle::mir::mono::MonoItem;
use cranelift_jit::{JITBuilder, JITModule};
use crate::prelude::*;
use crate::{prelude::*, BackendConfig};
use crate::{CodegenCx, CodegenMode};
thread_local! {
pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
}
pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
}
#[cfg(unix)]
unsafe {
// When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so
// __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again
// as global.
// FIXME remove once atomic_shim is gone
let mut dl_info: libc::Dl_info = std::mem::zeroed();
assert_ne!(
libc::dladdr(run_jit as *const libc::c_void, &mut dl_info),
0
);
assert_ne!(
libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL),
std::ptr::null_mut(),
);
}
let imported_symbols = load_imported_symbols_for_jit(tcx);
let mut jit_builder = JITBuilder::with_isa(
crate::build_isa(tcx.sess),
cranelift_module::default_libcall_names(),
);
jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
let mut jit_builder =
JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
jit_builder.symbols(imported_symbols);
let mut jit_module = JITModule::new(jit_builder);
assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
@ -56,14 +37,10 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
AbiParam::new(jit_module.target_config().pointer_type()),
AbiParam::new(jit_module.target_config().pointer_type()),
],
returns: vec![AbiParam::new(
jit_module.target_config().pointer_type(), /*isize*/
)],
returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let main_func_id = jit_module
.declare_function("main", Linkage::Import, &sig)
.unwrap();
let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
let mono_items = cgus
@ -74,19 +51,19 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
.into_iter()
.collect::<Vec<(_, (_, _))>>();
let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
super::time(tcx, "codegen mono items", || {
super::predefine_mono_items(&mut cx, &mono_items);
for (mono_item, (linkage, visibility)) in mono_items {
let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
match mono_item {
MonoItem::Fn(inst) => match codegen_mode {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
cx.tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, inst, linkage)
});
cx.tcx
.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
}
CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
},
@ -101,7 +78,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
}
});
let (mut jit_module, global_asm, _debug, mut unwind_context) =
let (global_asm, _debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
jit_module.finalize_definitions();
@ -109,7 +86,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
tcx.sess.fatal("Inline asm is not supported in JIT mode");
}
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
tcx.sess.abort_if_errors();
@ -120,7 +97,9 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
println!(
"Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
);
let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
unsafe { ::std::mem::transmute(finalized_main) };
@ -136,6 +115,9 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
// useful as some dynamic linkers use it as a marker to jump over.
argv.push(std::ptr::null());
BACKEND_CONFIG.with(|tls_backend_config| {
assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
});
CURRENT_MODULE
.with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
@ -153,21 +135,19 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
CURRENT_MODULE.with(|jit_module| {
let mut jit_module = jit_module.borrow_mut();
let jit_module = jit_module.as_mut().unwrap();
let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
let backend_config =
BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
let name = tcx.symbol_name(instance).name.to_string();
let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance);
let func_id = cx
.module
.declare_function(&name, Linkage::Export, &sig)
.unwrap();
cx.module.prepare_for_function_redefine(func_id).unwrap();
let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
jit_module.prepare_for_function_redefine(func_id).unwrap();
tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
});
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
tcx.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
let (global_asm, _debug_context, unwind_context) = cx.finalize();
assert!(global_asm.is_empty());
jit_module.finalize_definitions();
std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
@ -194,9 +174,8 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
Linkage::NotLinked | Linkage::IncludedFromDylib => {}
Linkage::Static => {
let name = tcx.crate_name(cnum);
let mut err = tcx
.sess
.struct_err(&format!("Can't load static lib {}", name.as_str()));
let mut err =
tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
err.emit();
}
@ -217,6 +196,11 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
return None;
}
if name.starts_with("rust_metadata_") {
// The metadata is part of a section that is not loaded by the dynamic linker in
// case of cg_llvm.
return None;
}
let dlsym_name = if cfg!(target_os = "macos") {
// On macOS `dlsym` expects the name without leading `_`.
assert!(name.starts_with('_'), "{:?}", name);
@ -236,17 +220,14 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
imported_symbols
}
pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
let tcx = cx.tcx;
let pointer_type = cx.module.target_config().pointer_type();
let name = tcx.symbol_name(inst).name.to_string();
let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
let func_id = cx
.module
.declare_function(&name, Linkage::Export, &sig)
.unwrap();
let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
let instance_ptr = Box::into_raw(Box::new(inst));
@ -267,28 +248,18 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In
let mut builder_ctx = FunctionBuilderContext::new();
let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
let jit_fn = cx
.module
.declare_func_in_func(jit_fn, trampoline_builder.func);
let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
let sig_ref = trampoline_builder.func.import_signature(sig);
let entry_block = trampoline_builder.create_block();
trampoline_builder.append_block_params_for_function_params(entry_block);
let fn_args = trampoline_builder
.func
.dfg
.block_params(entry_block)
.to_vec();
let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
trampoline_builder.switch_to_block(entry_block);
let instance_ptr = trampoline_builder
.ins()
.iconst(pointer_type, instance_ptr as u64 as i64);
let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
let call_inst = trampoline_builder
.ins()
.call_indirect(sig_ref, jitted_fn, &fn_args);
let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
trampoline_builder.ins().return_(&ret_vals);

32
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View File

@ -17,33 +17,30 @@ pub(crate) fn codegen_crate(
tcx: TyCtxt<'_>,
metadata: EncodedMetadata,
need_metadata_module: bool,
config: crate::BackendConfig,
backend_config: crate::BackendConfig,
) -> Box<dyn Any> {
tcx.sess.abort_if_errors();
match config.codegen_mode {
CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
match backend_config.codegen_mode {
CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
let is_executable = tcx
.sess
.crate_types()
.contains(&rustc_session::config::CrateType::Executable);
let is_executable =
tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
if !is_executable {
tcx.sess.fatal("can't jit non-executable crate");
}
#[cfg(feature = "jit")]
let _: ! = jit::run_jit(tcx, config.codegen_mode);
let _: ! = jit::run_jit(tcx, backend_config);
#[cfg(not(feature = "jit"))]
tcx.sess
.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
}
}
}
fn predefine_mono_items<'tcx>(
cx: &mut crate::CodegenCx<'tcx, impl Module>,
cx: &mut crate::CodegenCx<'_, 'tcx>,
mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
) {
cx.tcx.sess.time("predefine functions", || {
@ -63,21 +60,12 @@ fn predefine_mono_items<'tcx>(
}
fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
.as_ref()
.map(|val| &**val)
== Ok("1")
{
if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
let before = std::time::Instant::now();
let res = tcx.sess.time(name, f);
let after = std::time::Instant::now();
println!(
"[{:<30}: {}] end time: {:?}",
tcx.crate_name(LOCAL_CRATE),
name,
after - before
);
println!("[{:<30}: {}] end time: {:?}", tcx.crate_name(LOCAL_CRATE), name, after - before);
res
} else {
tcx.sess.time(name, f)

57
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View File

@ -9,7 +9,7 @@ use rustc_middle::mir::InlineAsmOperand;
use rustc_target::asm::*;
pub(crate) fn codegen_inline_asm<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
_span: Span,
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
@ -53,11 +53,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
crate::base::codegen_operand(fx, value).load_scalar(fx),
));
}
InlineAsmOperand::Out {
reg,
late: _,
place,
} => {
InlineAsmOperand::Out { reg, late: _, place } => {
let reg = expect_reg(reg);
clobbered_regs.push((reg, new_slot(reg.reg_class())));
if let Some(place) = place {
@ -68,12 +64,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
));
}
}
InlineAsmOperand::InOut {
reg,
late: _,
ref in_value,
out_place,
} => {
InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => {
let reg = expect_reg(reg);
clobbered_regs.push((reg, new_slot(reg.reg_class())));
inputs.push((
@ -97,11 +88,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let inline_asm_index = fx.inline_asm_index;
fx.inline_asm_index += 1;
let asm_name = format!(
"{}__inline_asm_{}",
fx.tcx.symbol_name(fx.instance).name,
inline_asm_index
);
let asm_name =
format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index);
let generated_asm = generate_asm_wrapper(
&asm_name,
@ -129,12 +117,7 @@ fn generate_asm_wrapper(
let mut generated_asm = String::new();
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
writeln!(
generated_asm,
".section .text.{},\"ax\",@progbits",
asm_name
)
.unwrap();
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
generated_asm.push_str(".intel_syntax noprefix\n");
@ -164,11 +147,7 @@ fn generate_asm_wrapper(
InlineAsmTemplatePiece::String(s) => {
generated_asm.push_str(s);
}
InlineAsmTemplatePiece::Placeholder {
operand_idx: _,
modifier: _,
span: _,
} => todo!(),
InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(),
}
}
generated_asm.push('\n');
@ -203,7 +182,7 @@ fn generate_asm_wrapper(
}
fn call_inline_asm<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
asm_name: &str,
slot_size: Size,
inputs: Vec<(InlineAsmReg, Size, Value)>,
@ -230,17 +209,12 @@ fn call_inline_asm<'tcx>(
},
)
.unwrap();
let inline_asm_func = fx
.cx
.module
.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(inline_asm_func, asm_name);
for (_reg, offset, value) in inputs {
fx.bcx
.ins()
.stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
}
let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
@ -248,10 +222,7 @@ fn call_inline_asm<'tcx>(
for (_reg, offset, place) in outputs {
let ty = fx.clif_type(place.layout().ty).unwrap();
let value = fx
.bcx
.ins()
.stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
}
}
@ -267,8 +238,7 @@ fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsm
match arch {
InlineAsmArch::X86_64 => {
write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None)
.unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
generated_asm.push('\n');
}
_ => unimplemented!("save_register for {:?}", arch),
@ -284,8 +254,7 @@ fn restore_register(
match arch {
InlineAsmArch::X86_64 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86_64, None)
.unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
}
_ => unimplemented!("restore_register for {:?}", arch),

44
compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
View File

@ -6,7 +6,7 @@ use crate::prelude::*;
///
/// This emulates an intel cpu with sse and sse2 support, but which doesn't support anything else.
pub(crate) fn codegen_cpuid_call<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
leaf: Value,
_subleaf: Value,
) -> (Value, Value, Value, Value) {
@ -31,54 +31,28 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
fx.bcx.switch_to_block(leaf_0);
let max_basic_leaf = fx.bcx.ins().iconst(types::I32, 1);
let vend0 = fx
.bcx
.ins()
.iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
let vend2 = fx
.bcx
.ins()
.iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
let vend1 = fx
.bcx
.ins()
.iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
fx.bcx
.ins()
.jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
let vend0 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
let vend2 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
let vend1 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
fx.bcx.ins().jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
fx.bcx.switch_to_block(leaf_1);
let cpu_signature = fx.bcx.ins().iconst(types::I32, 0);
let additional_information = fx.bcx.ins().iconst(types::I32, 0);
let ecx_features = fx.bcx.ins().iconst(types::I32, 0);
let edx_features = fx
.bcx
.ins()
.iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
fx.bcx.ins().jump(
dest,
&[
cpu_signature,
additional_information,
ecx_features,
edx_features,
],
);
let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]);
fx.bcx.switch_to_block(leaf_8000_0000);
let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0);
let zero = fx.bcx.ins().iconst(types::I32, 0);
fx.bcx
.ins()
.jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
fx.bcx.ins().jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
fx.bcx.switch_to_block(leaf_8000_0001);
let zero = fx.bcx.ins().iconst(types::I32, 0);
let proc_info_ecx = fx.bcx.ins().iconst(types::I32, 0);
let proc_info_edx = fx.bcx.ins().iconst(types::I32, 0);
fx.bcx
.ins()
.jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
fx.bcx.switch_to_block(unsupported_leaf);
crate::trap::trap_unreachable(

2
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View File

@ -6,7 +6,7 @@ use crate::prelude::*;
use rustc_middle::ty::subst::SubstsRef;
pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,
substs: SubstsRef<'tcx>,
args: &[mir::Operand<'tcx>],

280
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View File

@ -9,6 +9,7 @@ pub(crate) use cpuid::codegen_cpuid_call;
pub(crate) use llvm::codegen_llvm_intrinsic_call;
use crate::prelude::*;
use cranelift_codegen::ir::AtomicRmwOp;
use rustc_middle::ty::print::with_no_trimmed_paths;
macro intrinsic_pat {
@ -112,38 +113,6 @@ macro call_intrinsic_match {
}
}
macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
crate::atomic_shim::lock_global_lock($fx);
let clif_ty = $fx.clif_type($T).unwrap();
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
let new = $fx.bcx.ins().$op(old, $src);
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
$ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
crate::atomic_shim::unlock_global_lock($fx);
}
macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
crate::atomic_shim::lock_global_lock($fx);
// Read old
let clif_ty = $fx.clif_type($T).unwrap();
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
// Compare
let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
let new = $fx.bcx.ins().select(is_eq, old, $src);
// Write new
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
let ret_val = CValue::by_val(old, $ret.layout());
$ret.write_cvalue($fx, ret_val);
crate::atomic_shim::unlock_global_lock($fx);
}
macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
match $ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -184,12 +153,12 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
}
}
fn simd_for_each_lane<'tcx, M: Module>(
fx: &mut FunctionCx<'_, 'tcx, M>,
fn simd_for_each_lane<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
val: CValue<'tcx>,
ret: CPlace<'tcx>,
f: impl Fn(
&mut FunctionCx<'_, 'tcx, M>,
&mut FunctionCx<'_, '_, 'tcx>,
TyAndLayout<'tcx>,
TyAndLayout<'tcx>,
Value,
@ -213,13 +182,13 @@ fn simd_for_each_lane<'tcx, M: Module>(
}
}
fn simd_pair_for_each_lane<'tcx, M: Module>(
fx: &mut FunctionCx<'_, 'tcx, M>,
fn simd_pair_for_each_lane<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
x: CValue<'tcx>,
y: CValue<'tcx>,
ret: CPlace<'tcx>,
f: impl Fn(
&mut FunctionCx<'_, 'tcx, M>,
&mut FunctionCx<'_, '_, 'tcx>,
TyAndLayout<'tcx>,
TyAndLayout<'tcx>,
Value,
@ -246,11 +215,11 @@ fn simd_pair_for_each_lane<'tcx, M: Module>(
}
}
fn simd_reduce<'tcx, M: Module>(
fx: &mut FunctionCx<'_, 'tcx, M>,
fn simd_reduce<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
val: CValue<'tcx>,
ret: CPlace<'tcx>,
f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
) {
let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
let lane_layout = fx.layout_of(lane_ty);
@ -258,20 +227,19 @@ fn simd_reduce<'tcx, M: Module>(
let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
for lane_idx in 1..lane_count {
let lane = val
.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
.load_scalar(fx);
let lane =
val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
res_val = f(fx, lane_layout, res_val, lane);
}
let res = CValue::by_val(res_val, lane_layout);
ret.write_cvalue(fx, res);
}
fn simd_reduce_bool<'tcx, M: Module>(
fx: &mut FunctionCx<'_, 'tcx, M>,
fn simd_reduce_bool<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
val: CValue<'tcx>,
ret: CPlace<'tcx>,
f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
) {
let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
assert!(ret.layout().ty.is_bool());
@ -279,9 +247,8 @@ fn simd_reduce_bool<'tcx, M: Module>(
let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
for lane_idx in 1..lane_count {
let lane = val
.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
.load_scalar(fx);
let lane =
val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
res_val = f(fx, res_val, lane);
}
@ -290,7 +257,7 @@ fn simd_reduce_bool<'tcx, M: Module>(
}
fn bool_to_zero_or_max_uint<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
val: Value,
) -> CValue<'tcx> {
@ -424,7 +391,7 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
}
pub(crate) fn codegen_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
instance: Instance<'tcx>,
args: &[mir::Operand<'tcx>],
destination: Option<(CPlace<'tcx>, BasicBlock)>,
@ -912,136 +879,175 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
};
_ if intrinsic.starts_with("atomic_fence"), () {
crate::atomic_shim::lock_global_lock(fx);
crate::atomic_shim::unlock_global_lock(fx);
fx.bcx.ins().fence();
};
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {
crate::atomic_shim::lock_global_lock(fx);
crate::atomic_shim::unlock_global_lock(fx);
// FIXME use a compiler fence once Cranelift supports it
fx.bcx.ins().fence();
};
_ if intrinsic.starts_with("atomic_load"), (c ptr) {
crate::atomic_shim::lock_global_lock(fx);
_ if intrinsic.starts_with("atomic_load"), <T> (v ptr) {
validate_atomic_type!(fx, intrinsic, span, T);
let ty = fx.clif_type(T).unwrap();
let inner_layout =
fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
let val = fx.bcx.ins().atomic_load(ty, MemFlags::trusted(), ptr);
let val = CValue::by_val(val, fx.layout_of(T));
ret.write_cvalue(fx, val);
crate::atomic_shim::unlock_global_lock(fx);
};
_ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
crate::atomic_shim::lock_global_lock(fx);
let val = val.load_scalar(fx);
let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
dest.write_cvalue(fx, val);
crate::atomic_shim::unlock_global_lock(fx);
fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
};
_ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, T);
_ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) {
let layout = new.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
crate::atomic_shim::lock_global_lock(fx);
let new = new.load_scalar(fx);
// Read old
let clif_ty = fx.clif_type(T).unwrap();
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
// Write new
let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
dest.write_cvalue(fx, src);
crate::atomic_shim::unlock_global_lock(fx);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
validate_atomic_type!(fx, intrinsic, span, T);
_ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
let layout = new.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let test_old = test_old.load_scalar(fx);
let new = new.load_scalar(fx);
crate::atomic_shim::lock_global_lock(fx);
// Read old
let clif_ty = fx.clif_type(T).unwrap();
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
// Compare
let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
// Write new
fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
ret.write_cvalue(fx, ret_val);
crate::atomic_shim::unlock_global_lock(fx);
ret.write_cvalue(fx, ret_val)
};
_ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) {
let layout = amount.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let amount = amount.load_scalar(fx);
atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) {
let layout = amount.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let amount = amount.load_scalar(fx);
atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
let src = src.load_scalar(fx);
atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
};
_ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, T);
_ if intrinsic.starts_with("atomic_and"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
crate::atomic_shim::lock_global_lock(fx);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
let clif_ty = fx.clif_type(T).unwrap();
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
let and = fx.bcx.ins().band(old, src);
let new = fx.bcx.ins().bnot(and);
fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_or"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
crate::atomic_shim::unlock_global_lock(fx);
};
_ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
let src = src.load_scalar(fx);
atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
// FIXME https://github.com/bytecodealliance/wasmtime/issues/2647
_ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_max"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
_ if intrinsic.starts_with("atomic_min"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
let src = src.load_scalar(fx);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
minnumf32, (v a, v b) {

2
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View File

@ -4,7 +4,7 @@ use super::*;
use crate::prelude::*;
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
instance: Instance<'tcx>,
args: &[mir::Operand<'tcx>],
ret: CPlace<'tcx>,

55
compiler/rustc_codegen_cranelift/src/lib.rs
View File

@ -11,8 +11,6 @@
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
#[cfg(feature = "jit")]
extern crate libc;
extern crate snap;
#[macro_use]
extern crate rustc_middle;
@ -53,7 +51,6 @@ mod abi;
mod allocator;
mod analyze;
mod archive;
mod atomic_shim;
mod backend;
mod base;
mod cast;
@ -129,9 +126,9 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
}
}
struct CodegenCx<'tcx, M: Module> {
struct CodegenCx<'m, 'tcx: 'm> {
tcx: TyCtxt<'tcx>,
module: M,
module: &'m mut dyn Module,
global_asm: String,
constants_cx: ConstantCx,
cached_context: Context,
@ -140,14 +137,20 @@ struct CodegenCx<'tcx, M: Module> {
unwind_context: UnwindContext<'tcx>,
}
impl<'tcx, M: Module> CodegenCx<'tcx, M> {
fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self {
let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame);
let debug_context = if debug_info {
Some(DebugContext::new(tcx, module.isa()))
} else {
None
};
impl<'m, 'tcx> CodegenCx<'m, 'tcx> {
fn new(
tcx: TyCtxt<'tcx>,
backend_config: BackendConfig,
module: &'m mut dyn Module,
debug_info: bool,
) -> Self {
let unwind_context = UnwindContext::new(
tcx,
module.isa(),
matches!(backend_config.codegen_mode, CodegenMode::Aot),
);
let debug_context =
if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None };
CodegenCx {
tcx,
module,
@ -160,14 +163,9 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> {
}
}
fn finalize(mut self) -> (M, String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
self.constants_cx.finalize(self.tcx, &mut self.module);
(
self.module,
self.global_asm,
self.debug_context,
self.unwind_context,
)
fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
self.constants_cx.finalize(self.tcx, self.module);
(self.global_asm, self.debug_context, self.unwind_context)
}
}
@ -302,14 +300,7 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
flags_builder.enable("is_pic").unwrap();
flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
flags_builder
.set(
"enable_verifier",
if cfg!(debug_assertions) {
"true"
} else {
"false"
},
)
.set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
.unwrap();
let tls_model = match target_triple.binary_format {
@ -338,11 +329,7 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
let flags = settings::Flags::new(flags_builder);
let variant = if cfg!(feature = "oldbe") {
cranelift_codegen::isa::BackendVariant::Legacy
} else {
cranelift_codegen::isa::BackendVariant::MachInst
};
let variant = cranelift_codegen::isa::BackendVariant::MachInst;
let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
// Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
// is interpreted as `bsr`.

29
compiler/rustc_codegen_cranelift/src/main_shim.rs
View File

@ -9,7 +9,6 @@ pub(crate) fn maybe_create_entry_wrapper(
tcx: TyCtxt<'_>,
module: &mut impl Module,
unwind_context: &mut UnwindContext<'_>,
use_jit: bool,
) {
let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
Some((def_id, entry_ty)) => (
@ -27,14 +26,7 @@ pub(crate) fn maybe_create_entry_wrapper(
return;
}
create_entry_fn(
tcx,
module,
unwind_context,
main_def_id,
use_start_lang_item,
use_jit,
);
create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item);
fn create_entry_fn(
tcx: TyCtxt<'_>,
@ -42,7 +34,6 @@ pub(crate) fn maybe_create_entry_wrapper(
unwind_context: &mut UnwindContext<'_>,
rust_main_def_id: DefId,
use_start_lang_item: bool,
use_jit: bool,
) {
let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
// Given that `main()` has no arguments,
@ -57,23 +48,17 @@ pub(crate) fn maybe_create_entry_wrapper(
AbiParam::new(m.target_config().pointer_type()),
AbiParam::new(m.target_config().pointer_type()),
],
returns: vec![AbiParam::new(
m.target_config().pointer_type(), /*isize*/
)],
returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
call_conv: CallConv::triple_default(m.isa().triple()),
};
let cmain_func_id = m
.declare_function("main", Linkage::Export, &cmain_sig)
.unwrap();
let cmain_func_id = m.declare_function("main", Linkage::Export, &cmain_sig).unwrap();
let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
let main_name = tcx.symbol_name(instance).name.to_string();
let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
let main_func_id = m
.declare_function(&main_name, Linkage::Import, &main_sig)
.unwrap();
let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
@ -86,8 +71,6 @@ pub(crate) fn maybe_create_entry_wrapper(
let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
crate::atomic_shim::init_global_lock(m, &mut bcx, use_jit);
let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
let call_inst = if use_start_lang_item {
@ -103,9 +86,7 @@ pub(crate) fn maybe_create_entry_wrapper(
.polymorphize(tcx);
let start_func_id = import_function(tcx, m, start_instance);
let main_val = bcx
.ins()
.func_addr(m.target_config().pointer_type(), main_func_ref);
let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv])

4
compiler/rustc_codegen_cranelift/src/metadata.rs
View File

@ -94,9 +94,7 @@ pub(crate) fn write_metadata<P: WriteMetadata>(
assert!(kind == MetadataKind::Compressed);
let mut compressed = tcx.metadata_encoding_version();
FrameEncoder::new(&mut compressed)
.write_all(&metadata.raw_data)
.unwrap();
FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
product.add_rustc_section(
rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),

72
compiler/rustc_codegen_cranelift/src/num.rs
View File

@ -41,7 +41,7 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
}
fn codegen_compare_bin_op<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
signed: bool,
lhs: Value,
@ -54,7 +54,7 @@ fn codegen_compare_bin_op<'tcx>(
}
pub(crate) fn codegen_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -93,17 +93,12 @@ pub(crate) fn codegen_binop<'tcx>(
ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
_ => unreachable!(
"{:?}({:?}, {:?})",
bin_op,
in_lhs.layout().ty,
in_rhs.layout().ty
),
_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
}
}
pub(crate) fn codegen_bool_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -124,7 +119,7 @@ pub(crate) fn codegen_bool_binop<'tcx>(
}
pub(crate) fn codegen_int_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -185,19 +180,14 @@ pub(crate) fn codegen_int_binop<'tcx>(
}
}
// Compare binops handles by `codegen_binop`.
_ => unreachable!(
"{:?}({:?}, {:?})",
bin_op,
in_lhs.layout().ty,
in_rhs.layout().ty
),
_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
};
CValue::by_val(val, in_lhs.layout())
}
pub(crate) fn codegen_checked_int_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -268,9 +258,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
let val = fx.bcx.ins().imul(lhs, rhs);
let has_underflow =
fx.bcx
.ins()
.icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
let has_overflow = fx.bcx.ins().icmp_imm(
IntCC::SignedGreaterThan,
val,
@ -309,10 +297,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
let val = fx.bcx.ins().ishl(lhs, actual_shift);
let ty = fx.bcx.func.dfg.value_type(val);
let max_shift = i64::from(ty.bits()) - 1;
let has_overflow = fx
.bcx
.ins()
.icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
(val, has_overflow)
}
BinOp::Shr => {
@ -326,38 +311,20 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
};
let ty = fx.bcx.func.dfg.value_type(val);
let max_shift = i64::from(ty.bits()) - 1;
let has_overflow = fx
.bcx
.ins()
.icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
(val, has_overflow)
}
_ => bug!(
"binop {:?} on checked int/uint lhs: {:?} rhs: {:?}",
bin_op,
in_lhs,
in_rhs
),
_ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
};
let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);
// FIXME directly write to result place instead
let out_place = CPlace::new_stack_slot(
fx,
fx.layout_of(
fx.tcx
.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()),
),
);
let out_layout = out_place.layout();
out_place.write_cvalue(fx, CValue::by_val_pair(res, has_overflow, out_layout));
out_place.to_cvalue(fx)
let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
CValue::by_val_pair(res, has_overflow, out_layout)
}
pub(crate) fn codegen_float_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -402,7 +369,7 @@ pub(crate) fn codegen_float_binop<'tcx>(
}
pub(crate) fn codegen_ptr_binop<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
in_rhs: CValue<'tcx>,
@ -452,9 +419,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
let ptr_cmp =
fx.bcx
.ins()
.icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
let extra_cmp = fx.bcx.ins().icmp(
bin_op_to_intcc(bin_op, false).unwrap(),
lhs_extra,
@ -466,9 +431,6 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
_ => panic!("bin_op {:?} on ptr", bin_op),
};
CValue::by_val(
fx.bcx.ins().bint(types::I8, res),
fx.layout_of(fx.tcx.types.bool),
)
CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
}
}

10
compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs
View File

@ -15,10 +15,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
// bytecodealliance/cranelift#1339 is implemented.
let mut block_insts = FxHashMap::default();
for block in cold_blocks
.keys()
.filter(|&block| cold_blocks.contains(block))
{
for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>();
for &inst in &insts {
ctx.func.layout.remove_inst(inst);
@ -28,10 +25,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
}
// And then append them at the back again.
for block in cold_blocks
.keys()
.filter(|&block| cold_blocks.contains(block))
{
for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
ctx.func.layout.append_block(block);
for inst in block_insts.remove(&block).unwrap() {
ctx.func.layout.append_inst(inst, block);

7
compiler/rustc_codegen_cranelift/src/optimize/mod.rs
View File

@ -19,7 +19,12 @@ pub(crate) fn optimize_function<'tcx>(
if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No {
return; // FIXME classify optimizations over opt levels
}
self::stack2reg::optimize_function(ctx, clif_comments);
// FIXME(#1142) stack2reg miscompiles lewton
if false {
self::stack2reg::optimize_function(ctx, clif_comments);
}
crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments);
crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
}

22
compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
View File

@ -10,10 +10,7 @@ use cranelift_frontend::FunctionBuilder;
pub(crate) fn maybe_unwrap_bint(bcx: &mut FunctionBuilder<'_>, arg: Value) -> Value {
if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
match bcx.func.dfg[arg_inst] {
InstructionData::Unary {
opcode: Opcode::Bint,
arg,
} => arg,
InstructionData::Unary { opcode: Opcode::Bint, arg } => arg,
_ => arg,
}
} else {
@ -54,12 +51,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
match bcx.func.dfg[arg_inst] {
// This is the lowering of Rvalue::Not
InstructionData::Load {
opcode: Opcode::Load,
arg: ptr,
flags,
offset,
} => {
InstructionData::Load { opcode: Opcode::Load, arg: ptr, flags, offset } => {
// Using `load.i8 + uextend.i32` would legalize to `uload8 + ireduce.i8 +
// uextend.i32`. Just `uload8` is much faster.
match bcx.func.dfg.ctrl_typevar(arg_inst) {
@ -95,20 +87,14 @@ pub(crate) fn maybe_known_branch_taken(
};
match bcx.func.dfg[arg_inst] {
InstructionData::UnaryBool {
opcode: Opcode::Bconst,
imm,
} => {
InstructionData::UnaryBool { opcode: Opcode::Bconst, imm } => {
if test_zero {
Some(!imm)
} else {
Some(imm)
}
}
InstructionData::UnaryImm {
opcode: Opcode::Iconst,
imm,
} => {
InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
if test_zero {
Some(imm.bits() == 0)
} else {

49
compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
View File

@ -175,16 +175,14 @@ impl<'a> OptimizeContext<'a> {
}
}
OptimizeContext {
ctx,
stack_slot_usage_map,
}
OptimizeContext { ctx, stack_slot_usage_map }
}
}
pub(super) fn optimize_function(
ctx: &mut Context,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] clif_comments: &mut crate::pretty_clif::CommentWriter,
#[cfg_attr(not(debug_assertions), allow(unused_variables))]
clif_comments: &mut crate::pretty_clif::CommentWriter,
) {
combine_stack_addr_with_load_store(&mut ctx.func);
@ -296,12 +294,7 @@ fn combine_stack_addr_with_load_store(func: &mut Function) {
while let Some(_block) = cursor.next_block() {
while let Some(inst) = cursor.next_inst() {
match cursor.func.dfg[inst] {
InstructionData::Load {
opcode: Opcode::Load,
arg: addr,
flags: _,
offset,
} => {
InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => {
if cursor.func.dfg.ctrl_typevar(inst) == types::I128
|| cursor.func.dfg.ctrl_typevar(inst).is_vector()
{
@ -391,20 +384,14 @@ fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) {
stack_slot_users
.stack_addr
.drain_filter(|inst| {
stack_addr_load_insts_users
.get(inst)
.map(|users| users.is_empty())
.unwrap_or(true)
stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
})
.for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst));
stack_slot_users
.stack_load
.drain_filter(|inst| {
stack_addr_load_insts_users
.get(inst)
.map(|users| users.is_empty())
.unwrap_or(true)
stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
})
.for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst));
}
@ -415,11 +402,8 @@ fn try_get_stack_slot_and_offset_for_addr(
addr: Value,
) -> Option<(StackSlot, Offset32)> {
if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) {
if let InstructionData::StackLoad {
opcode: Opcode::StackAddr,
stack_slot,
offset,
} = func.dfg[addr_inst]
if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } =
func.dfg[addr_inst]
{
return Some((stack_slot, offset));
}
@ -437,16 +421,8 @@ enum SpatialOverlap {
fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) {
match func.dfg[inst] {
InstructionData::StackLoad {
opcode: Opcode::StackAddr,
stack_slot,
offset,
}
| InstructionData::StackLoad {
opcode: Opcode::StackLoad,
stack_slot,
offset,
}
InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset }
| InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset }
| InstructionData::StackStore {
opcode: Opcode::StackStore,
stack_slot,
@ -471,10 +447,7 @@ fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
}
let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into();
let dest_end: i64 = dest_offset
.try_add_i64(i64::from(dest_size))
.unwrap()
.into();
let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into();
if src_end <= dest_offset.into() || dest_end <= src_offset.into() {
return SpatialOverlap::No;
}

123
compiler/rustc_codegen_cranelift/src/pointer.rs
View File

@ -23,35 +23,20 @@ pub(crate) enum PointerBase {
impl Pointer {
pub(crate) fn new(addr: Value) -> Self {
Pointer {
base: PointerBase::Addr(addr),
offset: Offset32::new(0),
}
Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
}
pub(crate) fn stack_slot(stack_slot: StackSlot) -> Self {
Pointer {
base: PointerBase::Stack(stack_slot),
offset: Offset32::new(0),
}
Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) }
}
pub(crate) fn const_addr<'a, 'tcx>(
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
addr: i64,
) -> Self {
pub(crate) fn const_addr(fx: &mut FunctionCx<'_, '_, '_>, addr: i64) -> Self {
let addr = fx.bcx.ins().iconst(fx.pointer_type, addr);
Pointer {
base: PointerBase::Addr(addr),
offset: Offset32::new(0),
}
Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
}
pub(crate) fn dangling(align: Align) -> Self {
Pointer {
base: PointerBase::Dangling(align),
offset: Offset32::new(0),
}
Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
}
#[cfg(debug_assertions)]
@ -59,46 +44,28 @@ impl Pointer {
(self.base, self.offset)
}
pub(crate) fn get_addr<'a, 'tcx>(self, fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> Value {
pub(crate) fn get_addr(self, fx: &mut FunctionCx<'_, '_, '_>) -> Value {
match self.base {
PointerBase::Addr(base_addr) => {
let offset: i64 = self.offset.into();
if offset == 0 {
base_addr
} else {
fx.bcx.ins().iadd_imm(base_addr, offset)
}
if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm(base_addr, offset) }
}
PointerBase::Stack(stack_slot) => {
fx.bcx
.ins()
.stack_addr(fx.pointer_type, stack_slot, self.offset)
fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset)
}
PointerBase::Dangling(align) => {
fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
}
PointerBase::Dangling(align) => fx
.bcx
.ins()
.iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
}
}
pub(crate) fn offset<'a, 'tcx>(
self,
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
extra_offset: Offset32,
) -> Self {
pub(crate) fn offset(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Offset32) -> Self {
self.offset_i64(fx, extra_offset.into())
}
pub(crate) fn offset_i64<'a, 'tcx>(
self,
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
extra_offset: i64,
) -> Self {
pub(crate) fn offset_i64(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: i64) -> Self {
if let Some(new_offset) = self.offset.try_add_i64(extra_offset) {
Pointer {
base: self.base,
offset: new_offset,
}
Pointer { base: self.base, offset: new_offset }
} else {
let base_offset: i64 = self.offset.into();
if let Some(new_offset) = base_offset.checked_add(extra_offset) {
@ -107,16 +74,12 @@ impl Pointer {
PointerBase::Stack(stack_slot) => {
fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)
}
PointerBase::Dangling(align) => fx
.bcx
.ins()
.iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
PointerBase::Dangling(align) => {
fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
}
};
let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset);
Pointer {
base: PointerBase::Addr(addr),
offset: Offset32::new(0),
}
Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
} else {
panic!(
"self.offset ({}) + extra_offset ({}) not representable in i64",
@ -126,31 +89,22 @@ impl Pointer {
}
}
pub(crate) fn offset_value<'a, 'tcx>(
self,
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
extra_offset: Value,
) -> Self {
pub(crate) fn offset_value(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Value) -> Self {
match self.base {
PointerBase::Addr(addr) => Pointer {
base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
offset: self.offset,
},
PointerBase::Stack(stack_slot) => {
let base_addr = fx
.bcx
.ins()
.stack_addr(fx.pointer_type, stack_slot, self.offset);
let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset);
Pointer {
base: PointerBase::Addr(fx.bcx.ins().iadd(base_addr, extra_offset)),
offset: Offset32::new(0),
}
}
PointerBase::Dangling(align) => {
let addr = fx
.bcx
.ins()
.iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
let addr =
fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
Pointer {
base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
offset: self.offset,
@ -159,46 +113,21 @@ impl Pointer {
}
}
pub(crate) fn load<'a, 'tcx>(
self,
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
ty: Type,
flags: MemFlags,
) -> Value {
pub(crate) fn load(self, fx: &mut FunctionCx<'_, '_, '_>, ty: Type, flags: MemFlags) -> Value {
match self.base {
PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset),
PointerBase::Stack(stack_slot) => {
if ty == types::I128 || ty.is_vector() {
// WORKAROUND for stack_load.i128 and stack_load.iXxY not being implemented
let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
fx.bcx.ins().load(ty, flags, base_addr, self.offset)
} else {
fx.bcx.ins().stack_load(ty, stack_slot, self.offset)
}
}
PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset),
PointerBase::Dangling(_align) => unreachable!(),
}
}
pub(crate) fn store<'a, 'tcx>(
self,
fx: &mut FunctionCx<'a, 'tcx, impl Module>,
value: Value,
flags: MemFlags,
) {
pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlags) {
match self.base {
PointerBase::Addr(base_addr) => {
fx.bcx.ins().store(flags, value, base_addr, self.offset);
}
PointerBase::Stack(stack_slot) => {
let val_ty = fx.bcx.func.dfg.value_type(value);
if val_ty == types::I128 || val_ty.is_vector() {
// WORKAROUND for stack_store.i128 and stack_store.iXxY not being implemented
let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
fx.bcx.ins().store(flags, value, base_addr, self.offset);
} else {
fx.bcx.ins().stack_store(value, stack_slot, self.offset);
}
fx.bcx.ins().stack_store(value, stack_slot, self.offset);
}
PointerBase::Dangling(_align) => unreachable!(),
}

72
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View File

@ -79,20 +79,14 @@ impl CommentWriter {
vec![
format!("symbol {}", tcx.symbol_name(instance).name),
format!("instance {:?}", instance),
format!(
"abi {:?}",
FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])
),
format!("abi {:?}", FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
String::new(),
]
} else {
vec![]
};
CommentWriter {
global_comments,
entity_comments: FxHashMap::default(),
}
CommentWriter { global_comments, entity_comments: FxHashMap::default() }
}
}
@ -186,7 +180,7 @@ impl FuncWriter for &'_ CommentWriter {
}
#[cfg(debug_assertions)]
impl<M: Module> FunctionCx<'_, '_, M> {
impl FunctionCx<'_, '_, '_> {
pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
self.clif_comments.add_global_comment(comment);
}
@ -201,12 +195,7 @@ impl<M: Module> FunctionCx<'_, '_, M> {
}
pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
cfg!(debug_assertions)
|| tcx
.sess
.opts
.output_types
.contains_key(&OutputType::LlvmAssembly)
tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
}
pub(crate) fn write_ir_file<'tcx>(
@ -245,40 +234,33 @@ pub(crate) fn write_clif_file<'tcx>(
context: &cranelift_codegen::Context,
mut clif_comments: &CommentWriter,
) {
write_ir_file(
tcx,
&format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
|file| {
let value_ranges = isa.map(|isa| {
context
.build_value_labels_ranges(isa)
.expect("value location ranges")
});
write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| {
let value_ranges =
isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
let mut clif = String::new();
cranelift_codegen::write::decorate_function(
&mut clif_comments,
&mut clif,
&context.func,
&DisplayFunctionAnnotations {
isa: Some(&*crate::build_isa(tcx.sess)),
value_ranges: value_ranges.as_ref(),
},
)
.unwrap();
let mut clif = String::new();
cranelift_codegen::write::decorate_function(
&mut clif_comments,
&mut clif,
&context.func,
&DisplayFunctionAnnotations {
isa: Some(&*crate::build_isa(tcx.sess)),
value_ranges: value_ranges.as_ref(),
},
)
.unwrap();
writeln!(file, "test compile")?;
writeln!(file, "set is_pic")?;
writeln!(file, "set enable_simd")?;
writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
},
);
writeln!(file, "test compile")?;
writeln!(file, "set is_pic")?;
writeln!(file, "set enable_simd")?;
writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
});
}
impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> {
impl fmt::Debug for FunctionCx<'_, '_, '_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "{:?}", self.instance.substs)?;
writeln!(f, "{:?}", self.local_map)?;

15
compiler/rustc_codegen_cranelift/src/toolchain.rs
View File

@ -71,12 +71,9 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
flavor,
)),
(Some(linker), None) => {
let stem = linker
.file_stem()
.and_then(|stem| stem.to_str())
.unwrap_or_else(|| {
sess.fatal("couldn't extract file stem from specified linker")
});
let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| {
sess.fatal("couldn't extract file stem from specified linker")
});
let flavor = if stem == "emcc" {
LinkerFlavor::Em
@ -105,11 +102,7 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
// linker and linker flavor specified via command line have precedence over what the target
// specification specifies
if let Some(ret) = infer_from(
sess,
sess.opts.cg.linker.clone(),
sess.opts.cg.linker_flavor,
) {
if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
return ret;
}

12
compiler/rustc_codegen_cranelift/src/trap.rs
View File

@ -2,7 +2,7 @@
use crate::prelude::*;
fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
let puts = fx
.cx
.module
@ -29,7 +29,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
}
/// Trap code: user1
pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
fx.bcx.ins().trap(TrapCode::User(1));
}
@ -38,7 +38,7 @@ pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsR
/// so you can **not** add instructions to it afterwards.
///
/// Trap code: user65535
pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
@ -47,7 +47,7 @@ pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: im
///
/// Trap code: user65535
pub(crate) fn trap_unreachable_ret_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
dest_layout: TyAndLayout<'tcx>,
msg: impl AsRef<str>,
) -> CValue<'tcx> {
@ -62,7 +62,7 @@ pub(crate) fn trap_unreachable_ret_value<'tcx>(
/// to it afterwards.
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
@ -72,7 +72,7 @@ pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg:
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented_ret_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
dest_layout: TyAndLayout<'tcx>,
msg: impl AsRef<str>,
) -> CValue<'tcx> {

75
compiler/rustc_codegen_cranelift/src/unsize.rs
View File

@ -13,19 +13,18 @@ use crate::prelude::*;
/// in an upcast, where the new vtable for an object will be derived
/// from the old one.
pub(crate) fn unsized_info<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
source: Ty<'tcx>,
target: Ty<'tcx>,
old_info: Option<Value>,
) -> Value {
let (source, target) =
fx.tcx
.struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
fx.tcx.struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
match (&source.kind(), &target.kind()) {
(&ty::Array(_, len), &ty::Slice(_)) => fx.bcx.ins().iconst(
fx.pointer_type,
len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64,
),
(&ty::Array(_, len), &ty::Slice(_)) => fx
.bcx
.ins()
.iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
(&ty::Dynamic(..), &ty::Dynamic(..)) => {
// For now, upcasts are limited to changes in marker
// traits, and hence never actually require an actual
@ -35,17 +34,13 @@ pub(crate) fn unsized_info<'tcx>(
(_, &ty::Dynamic(ref data, ..)) => {
crate::vtable::get_vtable(fx, fx.layout_of(source), data.principal())
}
_ => bug!(
"unsized_info: invalid unsizing {:?} -> {:?}",
source,
target
),
_ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
}
}
/// Coerce `src` to `dst_ty`. `src_ty` must be a thin pointer.
fn unsize_thin_ptr<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
src: Value,
src_layout: TyAndLayout<'tcx>,
dst_layout: TyAndLayout<'tcx>,
@ -89,24 +84,22 @@ fn unsize_thin_ptr<'tcx>(
/// Coerce `src`, which is a reference to a value of type `src_ty`,
/// to a value of type `dst_ty` and store the result in `dst`
pub(crate) fn coerce_unsized_into<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
src: CValue<'tcx>,
dst: CPlace<'tcx>,
) {
let src_ty = src.layout().ty;
let dst_ty = dst.layout().ty;
let mut coerce_ptr = || {
let (base, info) = if fx
.layout_of(src.layout().ty.builtin_deref(true).unwrap().ty)
.is_unsized()
{
// fat-ptr to fat-ptr unsize preserves the vtable
// i.e., &'a fmt::Debug+Send => &'a fmt::Debug
src.load_scalar_pair(fx)
} else {
let base = src.load_scalar(fx);
unsize_thin_ptr(fx, base, src.layout(), dst.layout())
};
let (base, info) =
if fx.layout_of(src.layout().ty.builtin_deref(true).unwrap().ty).is_unsized() {
// fat-ptr to fat-ptr unsize preserves the vtable
// i.e., &'a fmt::Debug+Send => &'a fmt::Debug
src.load_scalar_pair(fx)
} else {
let base = src.load_scalar(fx);
unsize_thin_ptr(fx, base, src.layout(), dst.layout())
};
dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout()));
};
match (&src_ty.kind(), &dst_ty.kind()) {
@ -131,39 +124,26 @@ pub(crate) fn coerce_unsized_into<'tcx>(
}
}
}
_ => bug!(
"coerce_unsized_into: invalid coercion {:?} -> {:?}",
src_ty,
dst_ty
),
_ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty),
}
}
// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs
pub(crate) fn size_and_align_of_dst<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
info: Value,
) -> (Value, Value) {
if !layout.is_unsized() {
let size = fx
.bcx
.ins()
.iconst(fx.pointer_type, layout.size.bytes() as i64);
let align = fx
.bcx
.ins()
.iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
let size = fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64);
let align = fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
return (size, align);
}
match layout.ty.kind() {
ty::Dynamic(..) => {
// load size/align from vtable
(
crate::vtable::size_of_obj(fx, info),
crate::vtable::min_align_of_obj(fx, info),
)
(crate::vtable::size_of_obj(fx, info), crate::vtable::min_align_of_obj(fx, info))
}
ty::Slice(_) | ty::Str => {
let unit = layout.field(fx, 0);
@ -171,9 +151,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
// times the unit size.
(
fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64),
fx.bcx
.ins()
.iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
)
}
_ => {
@ -211,10 +189,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
// Choose max of two known alignments (combined value must
// be aligned according to more restrictive of the two).
let cmp = fx
.bcx
.ins()
.icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
let align = fx.bcx.ins().select(cmp, sized_align, unsized_align);
// Issue #27023: must add any necessary padding to `size`

187
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View File

@ -6,7 +6,7 @@ use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::immediates::Offset32;
fn codegen_field<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
base: Pointer,
extra: Option<Value>,
layout: TyAndLayout<'tcx>,
@ -15,11 +15,8 @@ fn codegen_field<'tcx>(
let field_offset = layout.fields.offset(field.index());
let field_layout = layout.field(&*fx, field.index());
let simple = |fx: &mut FunctionCx<'_, '_, _>| {
(
base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()),
field_layout,
)
let simple = |fx: &mut FunctionCx<'_, '_, '_>| {
(base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
};
if let Some(extra) = extra {
@ -58,10 +55,7 @@ fn scalar_pair_calculate_b_offset(
a_scalar: &Scalar,
b_scalar: &Scalar,
) -> Offset32 {
let b_offset = a_scalar
.value
.size(&tcx)
.align_to(b_scalar.value.align(&tcx).abi);
let b_offset = a_scalar.value.size(&tcx).align_to(b_scalar.value.align(&tcx).abi);
Offset32::new(b_offset.bytes().try_into().unwrap())
}
@ -106,10 +100,7 @@ impl<'tcx> CValue<'tcx> {
}
// FIXME remove
pub(crate) fn force_stack(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
) -> (Pointer, Option<Value>) {
pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) {
let layout = self.1;
match self.0 {
CValueInner::ByRef(ptr, meta) => (ptr, meta),
@ -129,7 +120,7 @@ impl<'tcx> CValue<'tcx> {
}
/// Load a value with layout.abi of scalar
pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> Value {
pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
let layout = self.1;
match self.0 {
CValueInner::ByRef(ptr, None) => {
@ -153,10 +144,7 @@ impl<'tcx> CValue<'tcx> {
}
/// Load a value pair with layout.abi of scalar pair
pub(crate) fn load_scalar_pair(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
) -> (Value, Value) {
pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
let layout = self.1;
match self.0 {
CValueInner::ByRef(ptr, None) => {
@ -183,7 +171,7 @@ impl<'tcx> CValue<'tcx> {
pub(crate) fn value_field(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
field: mir::Field,
) -> CValue<'tcx> {
let layout = self.1;
@ -219,21 +207,17 @@ impl<'tcx> CValue<'tcx> {
}
}
pub(crate) fn unsize_value(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
dest: CPlace<'tcx>,
) {
pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
crate::unsize::coerce_unsized_into(fx, self, dest);
}
/// If `ty` is signed, `const_val` must already be sign extended.
pub(crate) fn const_val(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
const_val: ty::ScalarInt,
) -> CValue<'tcx> {
assert_eq!(const_val.size(), layout.size);
assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
let clif_ty = fx.clif_type(layout.ty).unwrap();
@ -250,18 +234,11 @@ impl<'tcx> CValue<'tcx> {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
let const_val = const_val.to_bits(layout.size).unwrap();
let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64);
let msb = fx
.bcx
.ins()
.iconst(types::I64, (const_val >> 64) as u64 as i64);
let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64);
fx.bcx.ins().iconcat(lsb, msb)
}
ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..)
| ty::RawPtr(..) => {
fx
.bcx
.ins()
.iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
}
ty::Float(FloatTy::F32) => {
fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
@ -279,14 +256,8 @@ impl<'tcx> CValue<'tcx> {
}
pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self {
assert!(matches!(
self.layout().ty.kind(),
ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
));
assert!(matches!(
layout.ty.kind(),
ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
));
assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
assert_eq!(self.layout().abi, layout.abi);
CValue(self.0, layout)
}
@ -317,14 +288,11 @@ impl<'tcx> CPlace<'tcx> {
}
pub(crate) fn no_place(layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
CPlace {
inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None),
layout,
}
CPlace { inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), layout }
}
pub(crate) fn new_stack_slot(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
assert!(!layout.is_unsized());
@ -339,28 +307,22 @@ impl<'tcx> CPlace<'tcx> {
size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
offset: None,
});
CPlace {
inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None),
layout,
}
CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
}
pub(crate) fn new_var(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
local: Local,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
let var = Variable::with_u32(fx.next_ssa_var);
fx.next_ssa_var += 1;
fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap());
CPlace {
inner: CPlaceInner::Var(local, var),
layout,
}
CPlace { inner: CPlaceInner::Var(local, var), layout }
}
pub(crate) fn new_var_pair(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
local: Local,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
@ -372,17 +334,11 @@ impl<'tcx> CPlace<'tcx> {
let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap();
fx.bcx.declare_var(var1, ty1);
fx.bcx.declare_var(var2, ty2);
CPlace {
inner: CPlaceInner::VarPair(local, var1, var2),
layout,
}
CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout }
}
pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
CPlace {
inner: CPlaceInner::Addr(ptr, None),
layout,
}
CPlace { inner: CPlaceInner::Addr(ptr, None), layout }
}
pub(crate) fn for_ptr_with_extra(
@ -390,34 +346,27 @@ impl<'tcx> CPlace<'tcx> {
extra: Value,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
CPlace {
inner: CPlaceInner::Addr(ptr, Some(extra)),
layout,
}
CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout }
}
pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CValue<'tcx> {
pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> {
let layout = self.layout();
match self.inner {
CPlaceInner::Var(_local, var) => {
let val = fx.bcx.use_var(var);
fx.bcx
.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
//fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
CValue::by_val(val, layout)
}
CPlaceInner::VarPair(_local, var1, var2) => {
let val1 = fx.bcx.use_var(var1);
fx.bcx
.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
//fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
let val2 = fx.bcx.use_var(var2);
fx.bcx
.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
//fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
CValue::by_val_pair(val1, val2, layout)
}
CPlaceInner::VarLane(_local, var, lane) => {
let val = fx.bcx.use_var(var);
fx.bcx
.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
//fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
let val = fx.bcx.ins().extractlane(val, lane);
CValue::by_val(val, layout)
}
@ -447,11 +396,7 @@ impl<'tcx> CPlace<'tcx> {
}
}
pub(crate) fn write_cvalue(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
from: CValue<'tcx>,
) {
pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) {
assert_assignable(fx, from.layout().ty, self.layout().ty);
self.write_cvalue_maybe_transmute(fx, from, "write_cvalue");
@ -459,7 +404,7 @@ impl<'tcx> CPlace<'tcx> {
pub(crate) fn write_cvalue_transmute(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
from: CValue<'tcx>,
) {
self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute");
@ -467,12 +412,12 @@ impl<'tcx> CPlace<'tcx> {
fn write_cvalue_maybe_transmute(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
from: CValue<'tcx>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
) {
fn transmute_value<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
var: Variable,
data: Value,
dst_ty: Type,
@ -511,8 +456,7 @@ impl<'tcx> CPlace<'tcx> {
}
_ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
};
fx.bcx
.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
//fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
fx.bcx.def_var(var, data);
}
@ -558,15 +502,13 @@ impl<'tcx> CPlace<'tcx> {
// First get the old vector
let vector = fx.bcx.use_var(var);
fx.bcx
.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
//fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
// Next insert the written lane into the vector
let vector = fx.bcx.ins().insertlane(vector, data, lane);
// Finally write the new vector
fx.bcx
.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
//fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
fx.bcx.def_var(var, vector);
return;
@ -604,10 +546,7 @@ impl<'tcx> CPlace<'tcx> {
to_ptr.store(fx, val, flags);
}
CValueInner::ByValPair(_, _) => {
bug!(
"Non ScalarPair abi {:?} for ByValPair CValue",
dst_layout.abi
);
bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
}
CValueInner::ByRef(from_ptr, None) => {
let from_addr = from_ptr.get_addr(fx);
@ -632,7 +571,7 @@ impl<'tcx> CPlace<'tcx> {
pub(crate) fn place_field(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
field: mir::Field,
) -> CPlace<'tcx> {
let layout = self.layout();
@ -650,18 +589,8 @@ impl<'tcx> CPlace<'tcx> {
let layout = layout.field(&*fx, field.index());
match field.as_u32() {
0 => {
return CPlace {
inner: CPlaceInner::Var(local, var1),
layout,
}
}
1 => {
return CPlace {
inner: CPlaceInner::Var(local, var2),
layout,
}
}
0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout },
1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout },
_ => unreachable!("field should be 0 or 1"),
}
}
@ -680,7 +609,7 @@ impl<'tcx> CPlace<'tcx> {
pub(crate) fn place_index(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
index: Value,
) -> CPlace<'tcx> {
let (elem_layout, ptr) = match self.layout().ty.kind() {
@ -689,30 +618,24 @@ impl<'tcx> CPlace<'tcx> {
_ => bug!("place_index({:?})", self.layout().ty),
};
let offset = fx
.bcx
.ins()
.imul_imm(index, elem_layout.size.bytes() as i64);
let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64);
CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout)
}
pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CPlace<'tcx> {
pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> {
let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty);
if has_ptr_meta(fx.tcx, inner_layout.ty) {
let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx);
CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout)
} else {
CPlace::for_ptr(
Pointer::new(self.to_cvalue(fx).load_scalar(fx)),
inner_layout,
)
CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout)
}
}
pub(crate) fn place_ref(
self,
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
) -> CValue<'tcx> {
if has_ptr_meta(fx.tcx, self.layout().ty) {
@ -729,21 +652,18 @@ impl<'tcx> CPlace<'tcx> {
pub(crate) fn downcast_variant(
self,
fx: &FunctionCx<'_, 'tcx, impl Module>,
fx: &FunctionCx<'_, '_, 'tcx>,
variant: VariantIdx,
) -> Self {
assert!(!self.layout().is_unsized());
let layout = self.layout().for_variant(fx, variant);
CPlace {
inner: self.inner,
layout,
}
CPlace { inner: self.inner, layout }
}
}
#[track_caller]
pub(crate) fn assert_assignable<'tcx>(
fx: &FunctionCx<'_, 'tcx, impl Module>,
fx: &FunctionCx<'_, '_, 'tcx>,
from_ty: Ty<'tcx>,
to_ty: Ty<'tcx>,
) {
@ -776,12 +696,9 @@ pub(crate) fn assert_assignable<'tcx>(
}
(&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
for (from, to) in from_traits.iter().zip(to_traits) {
let from = fx
.tcx
.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
let to = fx
.tcx
.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
let from =
fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
assert_eq!(
from, to,
"Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",

23
compiler/rustc_codegen_cranelift/src/vtable.rs
View File

@ -15,7 +15,7 @@ fn vtable_memflags() -> MemFlags {
flags
}
pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
fx.bcx.ins().load(
pointer_ty(fx.tcx),
@ -25,7 +25,7 @@ pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: V
)
}
pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
fx.bcx.ins().load(
pointer_ty(fx.tcx),
@ -35,7 +35,7 @@ pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Valu
)
}
pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
fx.bcx.ins().load(
pointer_ty(fx.tcx),
@ -46,7 +46,7 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable:
}
pub(crate) fn get_ptr_and_method_ref<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
arg: CValue<'tcx>,
idx: usize,
) -> (Value, Value) {
@ -68,7 +68,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
}
pub(crate) fn get_vtable<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
) -> Value {
@ -85,7 +85,7 @@ pub(crate) fn get_vtable<'tcx>(
}
fn build_vtable<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
fx: &mut FunctionCx<'_, '_, 'tcx>,
layout: TyAndLayout<'tcx>,
trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
) -> DataId {
@ -94,7 +94,7 @@ fn build_vtable<'tcx>(
let drop_in_place_fn = import_function(
tcx,
&mut fx.cx.module,
fx.cx.module,
Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
);
@ -111,7 +111,7 @@ fn build_vtable<'tcx>(
opt_mth.map(|(def_id, substs)| {
import_function(
tcx,
&mut fx.cx.module,
fx.cx.module,
Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
.unwrap()
.polymorphize(fx.tcx),
@ -165,11 +165,8 @@ fn build_vtable<'tcx>(
}
fn write_usize(tcx: TyCtxt<'_>, buf: &mut [u8], idx: usize, num: u64) {
let pointer_size = tcx
.layout_of(ParamEnv::reveal_all().and(tcx.types.usize))
.unwrap()
.size
.bytes() as usize;
let pointer_size =
tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.usize)).unwrap().size.bytes() as usize;
let target = &mut buf[idx * pointer_size..(idx + 1) * pointer_size];
match tcx.data_layout.endian {

2
compiler/rustc_codegen_cranelift/test.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
set -e
./build.sh --sysroot none "$@"