Auto merge of #45920 - sunfishcode:trap-on-unreachable, r=Zoxc

Enable TrapUnreachable in LLVM.

This patch enables LLVM's TrapUnreachable flag, which tells it to translate `unreachable` instructions into hardware trap instructions, rather than allowing control flow to "fall through" into whatever code happens to follow it in memory.

This follows up on https://github.com/rust-lang/rust/issues/28728#issuecomment-332581533. For example, for @zackw's testcase [here](https://github.com/rust-lang/rust/issues/42009#issue-228745924), the output function contains a `ud2` instead of no code, so it won't "fall through" into whatever happens to be next in memory.

(I'm also working on the problem of LLVM optimizing away infinite loops, but the patch here is useful independently.)

I tested this patch on a few different codebases, and the code size increase ranged from 0.0% to 0.1%.

src/libcompiler_builtins

@@ -1 +1 @@
-Subproject commit 0b9844764ea1f99ea11a7917a4f3ba7fd2db775c
+Subproject commit f5532b22b5d741f3ea207b5b07e3e1ca63476f9b

src/librustc_back/target/mod.rs

@@ -435,6 +435,10 @@ pub struct TargetOptions {
 
     /// Default number of codegen units to use in debug mode
     pub default_codegen_units: Option<u64>,
+
+    /// Whether to generate trap instructions in places where optimization would
+    /// otherwise produce control flow that falls through into unrelated memory.
+    pub trap_unreachable: bool,
 }
 
 impl Default for TargetOptions {
@@ -498,6 +502,7 @@ impl Default for TargetOptions {
             stack_probes: false,
             min_global_align: None,
             default_codegen_units: None,
+            trap_unreachable: true,
         }
     }
 }
@@ -739,6 +744,7 @@ impl Target {
         key!(stack_probes, bool);
         key!(min_global_align, Option<u64>);
         key!(default_codegen_units, Option<u64>);
+        key!(trap_unreachable, bool);
 
         if let Some(array) = obj.find("abi-blacklist").and_then(Json::as_array) {
             for name in array.iter().filter_map(|abi| abi.as_string()) {
@@ -932,6 +938,7 @@ impl ToJson for Target {
         target_option_val!(stack_probes);
         target_option_val!(min_global_align);
         target_option_val!(default_codegen_units);
+        target_option_val!(trap_unreachable);
 
         if default.abi_blacklist != self.options.abi_blacklist {
             d.insert("abi-blacklist".to_string(), self.options.abi_blacklist.iter()

src/librustc_llvm/ffi.rs

@@ -1605,7 +1605,8 @@ extern "C" {
                                        UseSoftFP: bool,
                                        PositionIndependentExecutable: bool,
                                        FunctionSections: bool,
-                                       DataSections: bool)
+                                       DataSections: bool,
+                                       TrapUnreachable: bool)
                                        -> TargetMachineRef;
     pub fn LLVMRustDisposeTargetMachine(T: TargetMachineRef);
     pub fn LLVMRustAddAnalysisPasses(T: TargetMachineRef, PM: PassManagerRef, M: ModuleRef);

src/librustc_trans/back/write.rs

@@ -196,6 +196,7 @@ pub fn target_machine_factory(sess: &Session)
     let cpu = CString::new(cpu.as_bytes()).unwrap();
     let features = CString::new(target_feature(sess).as_bytes()).unwrap();
     let is_pie_binary = is_pie_binary(sess);
+    let trap_unreachable = sess.target.target.options.trap_unreachable;
 
     Arc::new(move || {
         let tm = unsafe {
@@ -208,6 +209,7 @@ pub fn target_machine_factory(sess: &Session)
                 is_pie_binary,
                 ffunction_sections,
                 fdata_sections,
+                trap_unreachable,
             )
         };
 

src/rustllvm/PassWrapper.cpp

@@ -366,7 +366,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
     LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
     bool PositionIndependentExecutable, bool FunctionSections,
-    bool DataSections) {
+    bool DataSections, bool TrapUnreachable) {
 
   auto CM = fromRust(RustCM);
   auto OptLevel = fromRust(RustOptLevel);
@@ -398,6 +398,14 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   Options.DataSections = DataSections;
   Options.FunctionSections = FunctionSections;
 
+  if (TrapUnreachable) {
+    // Tell LLVM to translate `unreachable` into an explicit trap instruction.
+    // This limits the extent of possible undefined behavior in some cases, as
+    // it prevents control flow from "falling through" into whatever code
+    // happens to be laid out next in memory.
+    Options.TrapUnreachable = true;
+  }
+
   TargetMachine *TM = TheTarget->createTargetMachine(
       Trip.getTriple(), RealCPU, Feature, Options, RM, CM, OptLevel);
   return wrap(TM);

src/test/run-make/intrinsic-unreachable/exit-ret.rs

@@ -11,10 +11,15 @@
 #![feature(asm)]
 #![crate_type="lib"]
 
-pub fn exit(n: usize) {
+#[deny(unreachable_code)]
+pub fn exit(n: usize) -> i32 {
     unsafe {
         // Pretend this asm is an exit() syscall.
         asm!("" :: "r"(n) :: "volatile");
         // Can't actually reach this point, but rustc doesn't know that.
     }
+    // This return value is just here to generate some extra code for a return
+    // value, making it easier for the test script to detect whether the
+    // compiler deleted it.
+    42
 }

src/test/run-make/intrinsic-unreachable/exit-unreachable.rs

@@ -13,10 +13,15 @@
 
 use std::intrinsics;
 
-pub fn exit(n: usize) -> ! {
+#[allow(unreachable_code)]
+pub fn exit(n: usize) -> i32 {
     unsafe {
         // Pretend this asm is an exit() syscall.
         asm!("" :: "r"(n) :: "volatile");
         intrinsics::unreachable()
     }
+    // This return value is just here to generate some extra code for a return
+    // value, making it easier for the test script to detect whether the
+    // compiler deleted it.
+    42
 }