Auto merge of #61408 - varkor:fmin-fmax-llvm-intrinsics, r=alexcrichton

Use LLVM intrinsics for floating-point min/max

Resurrection of https://github.com/rust-lang/rust/pull/46926, now that the optimisation issues are fixed. I've confirmed locally that https://github.com/rust-lang/rust/pull/61384 solves the issues.

I'm not sure if we're allowed to move the `min`/`max` methods from libcore to libstd: I can't quite tell what the status is from https://github.com/rust-lang/rust/issues/50145. However, this is necessary to use the intrinsics.

Fixes https://github.com/rust-lang/rust/issues/18384.

r? @SimonSapin
cc @rkruppe @nikic
This commit is contained in:
bors 2019-06-07 01:42:15 +00:00
commit c5295ac64a
8 changed files with 102 additions and 57 deletions

View File

@ -25,7 +25,7 @@ dependencies = [
name = "alloc"
version = "0.0.0"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
"rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_xorshift 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -118,7 +118,7 @@ dependencies = [
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"backtrace-sys 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
@ -130,7 +130,7 @@ version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
]
@ -357,7 +357,7 @@ name = "cfg-if"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
]
@ -486,7 +486,7 @@ dependencies = [
[[package]]
name = "compiler_builtins"
version = "0.1.15"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
@ -768,7 +768,7 @@ name = "dlmalloc"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
]
@ -934,7 +934,7 @@ name = "fortanix-sgx-abi"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
]
@ -1095,7 +1095,7 @@ name = "hashbrown"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-alloc 1.0.0",
"rustc-std-workspace-core 1.0.0",
]
@ -1803,7 +1803,7 @@ dependencies = [
name = "panic_abort"
version = "0.0.0"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1813,7 +1813,7 @@ name = "panic_unwind"
version = "0.0.0"
dependencies = [
"alloc 0.0.0",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
"unwind 0.0.0",
@ -1998,7 +1998,7 @@ name = "profiler_builtins"
version = "0.0.0"
dependencies = [
"cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
]
@ -2518,7 +2518,7 @@ name = "rustc-demangle"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-std-workspace-core 1.0.0",
]
@ -2646,7 +2646,7 @@ dependencies = [
"alloc 0.0.0",
"build_helper 0.1.0",
"cmake 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
]
@ -2878,7 +2878,7 @@ dependencies = [
"alloc 0.0.0",
"build_helper 0.1.0",
"cmake 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
]
@ -2941,7 +2941,7 @@ dependencies = [
"alloc 0.0.0",
"build_helper 0.1.0",
"cmake 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
]
@ -3060,7 +3060,7 @@ dependencies = [
"alloc 0.0.0",
"build_helper 0.1.0",
"cmake 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
]
@ -3338,7 +3338,7 @@ dependencies = [
"alloc 0.0.0",
"backtrace 0.3.29 (registry+https://github.com/rust-lang/crates.io-index)",
"cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
"dlmalloc 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fortanix-sgx-abi 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3936,7 +3936,7 @@ name = "unwind"
version = "0.0.0"
dependencies = [
"cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"core 0.0.0",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -4149,7 +4149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum colored 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b0aa3473e85a3161b59845d6096b289bb577874cafeaf75ea1b1beaa6572c7fc"
"checksum commoncrypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d056a8586ba25a1e4d61cb090900e495952c7886786fc55f909ab2f819b69007"
"checksum commoncrypto-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1fed34f46747aa73dfaa578069fd8279d2818ade2b55f38f22a9401c7f4083e2"
"checksum compiler_builtins 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "e899b947d7e71c3d35c0b6194d64025b84946640510e215090c815b20828964e"
"checksum compiler_builtins 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "e79ed19793c99771b386d76e08c3419409bb3d418b81a8b8afc73524247461cf"
"checksum compiletest_rs 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "f40ecc9332b68270998995c00f8051ee856121764a0d3230e64c9efd059d27b6"
"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e"
"checksum core-foundation 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4e2640d6d0bf22e82bed1b73c6aef8d5dd31e5abe6666c57e6d45e2649f4f887"

View File

@ -1051,6 +1051,19 @@ extern "rust-intrinsic" {
/// Returns the absolute value of an `f64`.
pub fn fabsf64(x: f64) -> f64;
/// Returns the minimum of two `f32` values.
#[cfg(not(bootstrap))]
pub fn minnumf32(x: f32, y: f32) -> f32;
/// Returns the minimum of two `f64` values.
#[cfg(not(bootstrap))]
pub fn minnumf64(x: f64, y: f64) -> f64;
/// Returns the maximum of two `f32` values.
#[cfg(not(bootstrap))]
pub fn maxnumf32(x: f32, y: f32) -> f32;
/// Returns the maximum of two `f64` values.
#[cfg(not(bootstrap))]
pub fn maxnumf64(x: f64, y: f64) -> f64;
/// Copies the sign from `y` to `x` for `f32` values.
pub fn copysignf32(x: f32, y: f32) -> f32;
/// Copies the sign from `y` to `x` for `f64` values.
@ -1561,3 +1574,47 @@ pub unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
pub unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize) {
real_intrinsics::write_bytes(dst, val, count)
}
// Simple bootstrap implementations of minnum/maxnum for stage0 compilation.
/// Returns the minimum of two `f32` values.
#[cfg(bootstrap)]
pub fn minnumf32(x: f32, y: f32) -> f32 {
// IEEE754 says: minNum(x, y) is the canonicalized number x if x < y, y if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signaling NaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if x < y || y != y { x } else { y }) * 1.0
}
/// Returns the minimum of two `f64` values.
#[cfg(bootstrap)]
pub fn minnumf64(x: f64, y: f64) -> f64 {
// Identical to the `f32` case.
(if x < y || y != y { x } else { y }) * 1.0
}
/// Returns the maximum of two `f32` values.
#[cfg(bootstrap)]
pub fn maxnumf32(x: f32, y: f32) -> f32 {
// IEEE754 says: maxNum(x, y) is the canonicalized number y if x < y, x if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signaling NaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if x < y || x != x { y } else { x }) * 1.0
}
/// Returns the maximum of two `f64` values.
#[cfg(bootstrap)]
pub fn maxnumf64(x: f64, y: f64) -> f64 {
// Identical to the `f32` case.
(if x < y || x != x { y } else { x }) * 1.0
}

View File

@ -7,6 +7,9 @@
#![stable(feature = "rust1", since = "1.0.0")]
#[cfg(not(test))]
use crate::intrinsics;
use crate::mem;
use crate::num::FpCategory;
@ -372,15 +375,7 @@ impl f32 {
#[stable(feature = "rust1", since = "1.0.0")]
#[inline]
pub fn max(self, other: f32) -> f32 {
// IEEE754 says: maxNum(x, y) is the canonicalized number y if x < y, x if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signalingNaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if self.is_nan() || self < other { other } else { self }) * 1.0
intrinsics::maxnumf32(self, other)
}
/// Returns the minimum of the two numbers.
@ -396,15 +391,7 @@ impl f32 {
#[stable(feature = "rust1", since = "1.0.0")]
#[inline]
pub fn min(self, other: f32) -> f32 {
// IEEE754 says: minNum(x, y) is the canonicalized number x if x < y, y if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signalingNaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if other.is_nan() || self < other { self } else { other }) * 1.0
intrinsics::minnumf32(self, other)
}
/// Raw transmutation to `u32`.

View File

@ -7,6 +7,9 @@
#![stable(feature = "rust1", since = "1.0.0")]
#[cfg(not(test))]
use crate::intrinsics;
use crate::mem;
use crate::num::FpCategory;
@ -385,15 +388,7 @@ impl f64 {
#[stable(feature = "rust1", since = "1.0.0")]
#[inline]
pub fn max(self, other: f64) -> f64 {
// IEEE754 says: maxNum(x, y) is the canonicalized number y if x < y, x if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signalingNaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if self.is_nan() || self < other { other } else { self }) * 1.0
intrinsics::maxnumf64(self, other)
}
/// Returns the minimum of the two numbers.
@ -409,15 +404,7 @@ impl f64 {
#[stable(feature = "rust1", since = "1.0.0")]
#[inline]
pub fn min(self, other: f64) -> f64 {
// IEEE754 says: minNum(x, y) is the canonicalized number x if x < y, y if y < x, the
// canonicalized number if one operand is a number and the other a quiet NaN. Otherwise it
// is either x or y, canonicalized (this means results might differ among implementations).
// When either x or y is a signalingNaN, then the result is according to 6.2.
//
// Since we do not support sNaN in Rust yet, we do not need to handle them.
// FIXME(nagisa): due to https://bugs.llvm.org/show_bug.cgi?id=33303 we canonicalize by
// multiplying by 1.0. Should switch to the `canonicalize` when it works.
(if other.is_nan() || self < other { self } else { other }) * 1.0
intrinsics::minnumf64(self, other)
}
/// Raw transmutation to `u64`.

View File

@ -645,6 +645,11 @@ impl CodegenCx<'b, 'tcx> {
ifn!("llvm.fabs.v4f64", fn(t_v4f64) -> t_v4f64);
ifn!("llvm.fabs.v8f64", fn(t_v8f64) -> t_v8f64);
ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.floor.f32", fn(t_f32) -> t_f32);
ifn!("llvm.floor.v2f32", fn(t_v2f32) -> t_v2f32);
ifn!("llvm.floor.v4f32", fn(t_v4f32) -> t_v4f32);

View File

@ -55,6 +55,10 @@ fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Valu
"fmaf64" => "llvm.fma.f64",
"fabsf32" => "llvm.fabs.f32",
"fabsf64" => "llvm.fabs.f64",
"minnumf32" => "llvm.minnum.f32",
"minnumf64" => "llvm.minnum.f64",
"maxnumf32" => "llvm.maxnum.f32",
"maxnumf64" => "llvm.maxnum.f64",
"copysignf32" => "llvm.copysign.f32",
"copysignf64" => "llvm.copysign.f64",
"floorf32" => "llvm.floor.f32",

View File

@ -70,7 +70,8 @@ pub fn intrisic_operation_unsafety(intrinsic: &str) -> hir::Unsafety {
"overflowing_add" | "overflowing_sub" | "overflowing_mul" |
"saturating_add" | "saturating_sub" |
"rotate_left" | "rotate_right" |
"ctpop" | "ctlz" | "cttz" | "bswap" | "bitreverse"
"ctpop" | "ctlz" | "cttz" | "bswap" | "bitreverse" |
"minnumf32" | "minnumf64" | "maxnumf32" | "maxnumf64"
=> hir::Unsafety::Normal,
_ => hir::Unsafety::Unsafe,
}
@ -272,6 +273,10 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
}
"fabsf32" => (0, vec![ tcx.types.f32 ], tcx.types.f32),
"fabsf64" => (0, vec![ tcx.types.f64 ], tcx.types.f64),
"minnumf32" => (0, vec![ tcx.types.f32, tcx.types.f32 ], tcx.types.f32),
"minnumf64" => (0, vec![ tcx.types.f64, tcx.types.f64 ], tcx.types.f64),
"maxnumf32" => (0, vec![ tcx.types.f32, tcx.types.f32 ], tcx.types.f32),
"maxnumf64" => (0, vec![ tcx.types.f64, tcx.types.f64 ], tcx.types.f64),
"copysignf32" => (0, vec![ tcx.types.f32, tcx.types.f32 ], tcx.types.f32),
"copysignf64" => (0, vec![ tcx.types.f64, tcx.types.f64 ], tcx.types.f64),
"floorf32" => (0, vec![ tcx.types.f32 ], tcx.types.f32),

View File

@ -19,7 +19,7 @@ panic_unwind = { path = "../libpanic_unwind", optional = true }
panic_abort = { path = "../libpanic_abort" }
core = { path = "../libcore" }
libc = { version = "0.2.51", default-features = false, features = ['rustc-dep-of-std'] }
compiler_builtins = { version = "0.1.15" }
compiler_builtins = { version = "0.1.16" }
profiler_builtins = { path = "../libprofiler_builtins", optional = true }
unwind = { path = "../libunwind" }
hashbrown = { version = "0.4.0", features = ['rustc-dep-of-std'] }