From f83901bb89e4043268ec3bf033b435d56e1ed303 Mon Sep 17 00:00:00 2001 From: Falco Hirschenberger Date: Fri, 21 Apr 2017 09:00:34 +0200 Subject: [PATCH] Adding support for the llvm `prefetch` intrinsic Related to #37251 --- src/libcore/intrinsics.rs | 47 ++++++++++++++++ src/librustc_trans/context.rs | 1 + src/librustc_trans/intrinsic.rs | 13 ++++- src/librustc_typeck/check/intrinsic.rs | 8 +++ src/test/codegen/prefetch.rs | 75 ++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 src/test/codegen/prefetch.rs diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 9f1870e56d3..3566bbdebc2 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -564,8 +564,55 @@ extern "rust-intrinsic" { pub fn atomic_umax_rel(dst: *mut T, src: T) -> T; pub fn atomic_umax_acqrel(dst: *mut T, src: T) -> T; pub fn atomic_umax_relaxed(dst: *mut T, src: T) -> T; + + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a noop. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache + #[cfg(not(stage0))] + pub fn prefetch_read_data(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a noop. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache + #[cfg(not(stage0))] + pub fn prefetch_write_data(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a noop. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache + #[cfg(not(stage0))] + pub fn prefetch_read_instruction(data: *const T, locality: i32); + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction + /// if supported; otherwise, it is a noop. + /// Prefetches have no effect on the behavior of the program but can change its performance + /// characteristics. + /// + /// The `locality` argument must be a constant integer and is a temporal locality specifier + /// ranging from (0) - no locality, to (3) - extremely local keep in cache + #[cfg(not(stage0))] + pub fn prefetch_write_instruction(data: *const T, locality: i32); } +// Empty bootstrap implementations for stage0 compilation +#[cfg(stage0)] +pub fn prefetch_read_data(_data: *const T, _locality: i32) { /* EMPTY */ } +#[cfg(stage0)] +pub fn prefetch_write_data(_data: *const T, _locality: i32) { /* EMPTY */ } +#[cfg(stage0)] +pub fn prefetch_read_instruction(_data: *const T, _locality: i32) { /* EMPTY */ } +#[cfg(stage0)] +pub fn prefetch_write_instruction(_data: *const T, _locality: i32) { /* EMPTY */ } + extern "rust-intrinsic" { pub fn atomic_fence(); diff --git a/src/librustc_trans/context.rs b/src/librustc_trans/context.rs index c3b16c2d07d..0413b0ea5c8 100644 --- a/src/librustc_trans/context.rs +++ b/src/librustc_trans/context.rs @@ -942,6 +942,7 @@ fn declare_intrinsic(ccx: &CrateContext, key: &str) -> Option { ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p); ifn!("llvm.assume", fn(i1) -> void); + ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); if ccx.sess().opts.debuginfo != NoDebugInfo { ifn!("llvm.dbg.declare", fn(Type::metadata(ccx), Type::metadata(ccx)) -> void); diff --git a/src/librustc_trans/intrinsic.rs b/src/librustc_trans/intrinsic.rs index 54e20f590c6..f2e6aa3ef00 100644 --- a/src/librustc_trans/intrinsic.rs +++ b/src/librustc_trans/intrinsic.rs @@ -255,7 +255,18 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bcx: &Builder<'a, 'tcx>, } C_nil(ccx) }, - + "prefetch_read_data" | "prefetch_write_data" | + "prefetch_read_instruction" | "prefetch_write_instruction" => { + let expect = ccx.get_intrinsic(&("llvm.prefetch")); + let (rw, cache_type) = match name { + "prefetch_read_data" => (0, 1), + "prefetch_write_data" => (1, 1), + "prefetch_read_instruction" => (0, 0), + "prefetch_write_instruction" => (1, 0), + _ => bug!() + }; + bcx.call(expect, &[llargs[0], C_i32(ccx, rw), llargs[1], C_i32(ccx, cache_type)], None) + }, "ctlz" | "cttz" | "ctpop" | "bswap" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" | "overflowing_add" | "overflowing_sub" | "overflowing_mul" | diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index 60067e6a6ec..daf202cd797 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -133,6 +133,14 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ], tcx.mk_nil()) } + "prefetch_read_data" | "prefetch_write_data" | + "prefetch_read_instruction" | "prefetch_write_instruction" => { + (1, vec![tcx.mk_ptr(ty::TypeAndMut { + ty: param(0), + mutbl: hir::MutImmutable + }), tcx.types.i32], + tcx.mk_nil()) + } "drop_in_place" => { (1, vec![tcx.mk_mut_ptr(param(0))], tcx.mk_nil()) } diff --git a/src/test/codegen/prefetch.rs b/src/test/codegen/prefetch.rs new file mode 100644 index 00000000000..9ca2f01f80d --- /dev/null +++ b/src/test/codegen/prefetch.rs @@ -0,0 +1,75 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::{prefetch_read_data, prefetch_write_data, + prefetch_read_instruction, prefetch_write_instruction}; + +#[no_mangle] +pub fn check_prefetch_read_data(data: &[i8]) { + unsafe { + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 1) + prefetch_read_data(data.as_ptr(), 0); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 1) + prefetch_read_data(data.as_ptr(), 1); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 1) + prefetch_read_data(data.as_ptr(), 2); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 1) + prefetch_read_data(data.as_ptr(), 3); + } +} + +#[no_mangle] +pub fn check_prefetch_write_data(data: &[i8]) { + unsafe { + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 1) + prefetch_write_data(data.as_ptr(), 0); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 1) + prefetch_write_data(data.as_ptr(), 1); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 1) + prefetch_write_data(data.as_ptr(), 2); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 1) + prefetch_write_data(data.as_ptr(), 3); + } +} + +#[no_mangle] +pub fn check_prefetch_read_instruction(data: &[i8]) { + unsafe { + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 0) + prefetch_read_instruction(data.as_ptr(), 0); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 0) + prefetch_read_instruction(data.as_ptr(), 1); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 0) + prefetch_read_instruction(data.as_ptr(), 2); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 0) + prefetch_read_instruction(data.as_ptr(), 3); + } +} + +#[no_mangle] +pub fn check_prefetch_write_instruction(data: &[i8]) { + unsafe { + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 0) + prefetch_write_instruction(data.as_ptr(), 0); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 0) + prefetch_write_instruction(data.as_ptr(), 1); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 0) + prefetch_write_instruction(data.as_ptr(), 2); + // CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 0) + prefetch_write_instruction(data.as_ptr(), 3); + } +} + +