Auto merge of #48346 - emilio:pgo, r=alexcrichton

Add basic PGO support.

This PR adds two mutually exclusive options for profile usage and generation using LLVM's instruction profile generation (the same as clang uses), `-C pgo-use` and `-C pgo-gen`.

See each commit for details.

src/libprofiler_builtins/build.rs

@@ -27,6 +27,7 @@ fn main() {
                                    "InstrProfilingFile.c",
                                    "InstrProfilingMerge.c",
                                    "InstrProfilingMergeFile.c",
+                                   "InstrProfilingNameVar.c",
                                    "InstrProfilingPlatformDarwin.c",
                                    "InstrProfilingPlatformLinux.c",
                                    "InstrProfilingPlatformOther.c",
@@ -42,6 +43,8 @@ fn main() {
         cfg.define("strdup", Some("_strdup"));
         cfg.define("open", Some("_open"));
         cfg.define("fdopen", Some("_fdopen"));
+        cfg.define("getpid", Some("_getpid"));
+        cfg.define("fileno", Some("_fileno"));
     } else {
         // Turn off various features of gcc and such, mostly copying
         // compiler-rt's build system already
@@ -50,6 +53,7 @@ fn main() {
         cfg.flag("-fomit-frame-pointer");
         cfg.flag("-ffreestanding");
         cfg.define("VISIBILITY_HIDDEN", None);
+        cfg.define("COMPILER_RT_HAS_UNAME", Some("1"));
     }
 
     for src in profile_sources {

src/librustc/session/config.rs

@@ -1249,6 +1249,14 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "extra arguments to prepend to the linker invocation (space separated)"),
     profile: bool = (false, parse_bool, [TRACKED],
                      "insert profiling code"),
+    pgo_gen: Option<String> = (None, parse_opt_string, [TRACKED],
+        "Generate PGO profile data, to a given file, or to the default \
+         location if it's empty."),
+    pgo_use: String = (String::new(), parse_string, [TRACKED],
+        "Use PGO profile data from the given profile file."),
+    disable_instrumentation_preinliner: bool =
+        (false, parse_bool, [TRACKED], "Disable the instrumentation pre-inliner, \
+        useful for profiling / PGO."),
     relro_level: Option<RelroLevel> = (None, parse_relro_level, [TRACKED],
         "choose which RELRO level to use"),
     nll: bool = (false, parse_bool, [UNTRACKED],
@@ -1773,6 +1781,13 @@ pub fn build_session_options_and_crate_config(
         );
     }
 
+    if debugging_opts.pgo_gen.is_some() && !debugging_opts.pgo_use.is_empty() {
+        early_error(
+            error_format,
+            "options `-Z pgo-gen` and `-Z pgo-use` are exclusive",
+        );
+    }
+
     let mut output_types = BTreeMap::new();
     if !debugging_opts.parse_only {
         for list in matches.opt_strs("emit") {
@@ -2886,6 +2901,14 @@ mod tests {
         opts.debugging_opts.tls_model = Some(String::from("tls model"));
         assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());
 
+        opts = reference.clone();
+        opts.debugging_opts.pgo_gen = Some(String::from("abc"));
+        assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash());
+
+        opts = reference.clone();
+        opts.debugging_opts.pgo_use = String::from("abc");
+        assert_ne!(reference.dep_tracking_hash(), opts.dep_tracking_hash());
+
         opts = reference.clone();
         opts.cg.metadata = vec![String::from("A"), String::from("B")];
         assert!(reference.dep_tracking_hash() != opts.dep_tracking_hash());

src/librustc_llvm/diagnostic.rs

@@ -121,6 +121,7 @@ impl InlineAsmDiagnostic {
 pub enum Diagnostic {
     Optimization(OptimizationDiagnostic),
     InlineAsm(InlineAsmDiagnostic),
+    PGO(DiagnosticInfoRef),
 
     /// LLVM has other types that we do not wrap here.
     UnknownDiagnostic(DiagnosticInfoRef),
@@ -160,6 +161,10 @@ impl Diagnostic {
                 Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di))
             }
 
+            Dk::PGOProfile => {
+                PGO(di)
+            }
+
             _ => UnknownDiagnostic(di),
         }
     }

src/librustc_llvm/ffi.rs

@@ -322,6 +322,7 @@ pub enum DiagnosticKind {
     OptimizationRemarkAnalysisAliasing,
     OptimizationRemarkOther,
     OptimizationFailure,
+    PGOProfile,
 }
 
 /// LLVMRustArchiveKind
@@ -1646,7 +1647,9 @@ extern "C" {
                                                OptLevel: CodeGenOptLevel,
                                                MergeFunctions: bool,
                                                SLPVectorize: bool,
-                                               LoopVectorize: bool);
+                                               LoopVectorize: bool,
+                                               PGOGenPath: *const c_char,
+                                               PGOUsePath: *const c_char);
     pub fn LLVMRustAddLibraryInfo(PM: PassManagerRef,
                                   M: ModuleRef,
                                   DisableSimplifyLibCalls: bool);
@@ -1741,6 +1744,7 @@ extern "C" {
     pub fn LLVMRustModuleCost(M: ModuleRef) -> u64;
 
     pub fn LLVMRustThinLTOAvailable() -> bool;
+    pub fn LLVMRustPGOAvailable() -> bool;
     pub fn LLVMRustWriteThinBitcodeToFile(PMR: PassManagerRef,
                                           M: ModuleRef,
                                           BC: *const c_char) -> bool;

src/librustc_metadata/creader.rs

@@ -784,7 +784,9 @@ impl<'a> CrateLoader<'a> {
     }
 
     fn inject_profiler_runtime(&mut self) {
-        if self.sess.opts.debugging_opts.profile {
+        if self.sess.opts.debugging_opts.profile ||
+            self.sess.opts.debugging_opts.pgo_gen.is_some()
+        {
             info!("loading profiler");
 
             let symbol = Symbol::intern("profiler_builtins");

src/librustc_trans/attributes.rs

@@ -92,6 +92,11 @@ pub fn set_probestack(cx: &CodegenCx, llfn: ValueRef) {
         _ => {}
     }
 
+    // probestack doesn't play nice either with pgo-gen.
+    if cx.sess().opts.debugging_opts.pgo_gen.is_some() {
+        return;
+    }
+
     // Flag our internal `__rust_probestack` function as the stack probe symbol.
     // This is defined in the `compiler-builtins` crate for each architecture.
     llvm::AddFunctionAttrStringValue(

src/librustc_trans/back/link.rs

@@ -1085,6 +1085,10 @@ fn link_args(cmd: &mut Linker,
         cmd.build_static_executable();
     }
 
+    if sess.opts.debugging_opts.pgo_gen.is_some() {
+        cmd.pgo_gen();
+    }
+
     // FIXME (#2397): At some point we want to rpath our guesses as to
     // where extern libraries might live, based on the
     // addl_lib_search_paths

src/librustc_trans/back/linker.rs

@@ -117,6 +117,7 @@ pub trait Linker {
     fn partial_relro(&mut self);
     fn no_relro(&mut self);
     fn optimize(&mut self);
+    fn pgo_gen(&mut self);
     fn debuginfo(&mut self);
     fn no_default_libraries(&mut self);
     fn build_dylib(&mut self, out_filename: &Path);
@@ -280,6 +281,24 @@ impl<'a> Linker for GccLinker<'a> {
         }
     }
 
+    fn pgo_gen(&mut self) {
+        if !self.sess.target.target.options.linker_is_gnu { return }
+
+        // If we're doing PGO generation stuff and on a GNU-like linker, use the
+        // "-u" flag to properly pull in the profiler runtime bits.
+        //
+        // This is because LLVM otherwise won't add the needed initialization
+        // for us on Linux (though the extra flag should be harmless if it
+        // does).
+        //
+        // See https://reviews.llvm.org/D14033 and https://reviews.llvm.org/D14030.
+        //
+        // Though it may be worth to try to revert those changes upstream, since
+        // the overhead of the initialization should be minor.
+        self.cmd.arg("-u");
+        self.cmd.arg("__llvm_profile_runtime");
+    }
+
     fn debuginfo(&mut self) {
         match self.sess.opts.debuginfo {
             DebugInfoLevel::NoDebugInfo => {
@@ -520,6 +539,10 @@ impl<'a> Linker for MsvcLinker<'a> {
         // Needs more investigation of `/OPT` arguments
     }
 
+    fn pgo_gen(&mut self) {
+        // Nothing needed here.
+    }
+
     fn debuginfo(&mut self) {
         // This will cause the Microsoft linker to generate a PDB file
         // from the CodeView line tables in the object files.
@@ -723,6 +746,10 @@ impl<'a> Linker for EmLinker<'a> {
         self.cmd.args(&["--memory-init-file", "0"]);
     }
 
+    fn pgo_gen(&mut self) {
+        // noop, but maybe we need something like the gnu linker?
+    }
+
     fn debuginfo(&mut self) {
         // Preserve names or generate source maps depending on debug info
         self.cmd.arg(match self.sess.opts.debuginfo {
@@ -888,6 +915,9 @@ impl Linker for WasmLd {
     fn optimize(&mut self) {
     }
 
+    fn pgo_gen(&mut self) {
+    }
+
     fn debuginfo(&mut self) {
     }
 

src/librustc_trans/back/symbol_export.rs

@@ -223,6 +223,20 @@ fn exported_symbols_provider_local<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         }
     }
 
+    if tcx.sess.opts.debugging_opts.pgo_gen.is_some() {
+        // These are weak symbols that point to the profile version and the
+        // profile name, which need to be treated as exported so LTO doesn't nix
+        // them.
+        const PROFILER_WEAK_SYMBOLS: [&'static str; 2] = [
+            "__llvm_profile_raw_version",
+            "__llvm_profile_filename",
+        ];
+        for sym in &PROFILER_WEAK_SYMBOLS {
+            let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(sym));
+            symbols.push((exported_symbol, SymbolExportLevel::C));
+        }
+    }
+
     if tcx.sess.crate_types.borrow().contains(&config::CrateTypeDylib) {
         let symbol_name = metadata_symbol_name(tcx);
         let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(&symbol_name));

src/librustc_trans/back/write.rs

@@ -240,6 +240,9 @@ pub struct ModuleConfig {
     /// Some(level) to optimize binary size, or None to not affect program size.
     opt_size: Option<llvm::CodeGenOptSize>,
 
+    pgo_gen: Option<String>,
+    pgo_use: String,
+
     // Flags indicating which outputs to produce.
     emit_no_opt_bc: bool,
     emit_bc: bool,
@@ -274,6 +277,9 @@ impl ModuleConfig {
             opt_level: None,
             opt_size: None,
 
+            pgo_gen: None,
+            pgo_use: String::new(),
+
             emit_no_opt_bc: false,
             emit_bc: false,
             emit_bc_compressed: false,
@@ -492,8 +498,13 @@ unsafe extern "C" fn diagnostic_handler(info: DiagnosticInfoRef, user: *mut c_vo
                                                 opt.message));
             }
         }
-
-        _ => (),
+        llvm::diagnostic::PGO(diagnostic_ref) => {
+            let msg = llvm::build_string(|s| {
+                llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
+            }).expect("non-UTF8 PGO diagnostic");
+            diag_handler.warn(&msg);
+        }
+        llvm::diagnostic::UnknownDiagnostic(..) => {},
     }
 }
 
@@ -932,6 +943,9 @@ pub fn start_async_translation(tcx: TyCtxt,
         modules_config.passes.push("insert-gcov-profiling".to_owned())
     }
 
+    modules_config.pgo_gen = sess.opts.debugging_opts.pgo_gen.clone();
+    modules_config.pgo_use = sess.opts.debugging_opts.pgo_use.clone();
+
     modules_config.opt_level = Some(get_llvm_opt_level(sess.opts.optimize));
     modules_config.opt_size = Some(get_llvm_opt_size(sess.opts.optimize));
 
@@ -2046,6 +2060,8 @@ pub unsafe fn with_llvm_pmb(llmod: ModuleRef,
                             config: &ModuleConfig,
                             opt_level: llvm::CodeGenOptLevel,
                             f: &mut FnMut(llvm::PassManagerBuilderRef)) {
+    use std::ptr;
+
     // Create the PassManagerBuilder for LLVM. We configure it with
     // reasonable defaults and prepare it to actually populate the pass
     // manager.
@@ -2053,11 +2069,27 @@ pub unsafe fn with_llvm_pmb(llmod: ModuleRef,
     let opt_size = config.opt_size.unwrap_or(llvm::CodeGenOptSizeNone);
     let inline_threshold = config.inline_threshold;
 
-    llvm::LLVMRustConfigurePassManagerBuilder(builder,
-                                              opt_level,
-                                              config.merge_functions,
-                                              config.vectorize_slp,
-                                              config.vectorize_loop);
+    let pgo_gen_path = config.pgo_gen.as_ref().map(|s| {
+        let s = if s.is_empty() { "default_%m.profraw" } else { s };
+        CString::new(s.as_bytes()).unwrap()
+    });
+
+    let pgo_use_path = if config.pgo_use.is_empty() {
+        None
+    } else {
+        Some(CString::new(config.pgo_use.as_bytes()).unwrap())
+    };
+
+    llvm::LLVMRustConfigurePassManagerBuilder(
+        builder,
+        opt_level,
+        config.merge_functions,
+        config.vectorize_slp,
+        config.vectorize_loop,
+        pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
+        pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
+    );
+
     llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);
 
     if opt_size != llvm::CodeGenOptSizeNone {

src/librustc_trans/base.rs

@@ -708,6 +708,13 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         }
     }
 
+    if (tcx.sess.opts.debugging_opts.pgo_gen.is_some() ||
+        !tcx.sess.opts.debugging_opts.pgo_use.is_empty()) &&
+        unsafe { !llvm::LLVMRustPGOAvailable() }
+    {
+        tcx.sess.fatal("this compiler's LLVM does not support PGO");
+    }
+
     let crate_hash = tcx.crate_hash(LOCAL_CRATE);
     let link_meta = link::build_link_meta(crate_hash);
 

src/librustc_trans/llvm_util.rs

@@ -61,6 +61,9 @@ unsafe fn configure_llvm(sess: &Session) {
         add("rustc"); // fake program name
         if sess.time_llvm_passes() { add("-time-passes"); }
         if sess.print_llvm_passes() { add("-debug-pass=Structure"); }
+        if sess.opts.debugging_opts.disable_instrumentation_preinliner {
+            add("-disable-preinline");
+        }
 
         for arg in &sess.opts.cg.llvm_args {
             add(&(*arg));

src/rustllvm/PassWrapper.cpp

@@ -44,6 +44,10 @@
 
 #include "llvm-c/Transforms/PassManagerBuilder.h"
 
+#if LLVM_VERSION_GE(4, 0)
+#define PGO_AVAILABLE
+#endif
+
 using namespace llvm;
 using namespace llvm::legacy;
 
@@ -428,12 +432,27 @@ extern "C" void LLVMRustAddAnalysisPasses(LLVMTargetMachineRef TM,
 
 extern "C" void LLVMRustConfigurePassManagerBuilder(
     LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel,
-    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize) {
+    bool MergeFunctions, bool SLPVectorize, bool LoopVectorize,
+    const char* PGOGenPath, const char* PGOUsePath) {
   // Ignore mergefunc for now as enabling it causes crashes.
   // unwrap(PMBR)->MergeFunctions = MergeFunctions;
   unwrap(PMBR)->SLPVectorize = SLPVectorize;
   unwrap(PMBR)->OptLevel = fromRust(OptLevel);
   unwrap(PMBR)->LoopVectorize = LoopVectorize;
+
+#ifdef PGO_AVAILABLE
+  if (PGOGenPath) {
+    assert(!PGOUsePath);
+    unwrap(PMBR)->EnablePGOInstrGen = true;
+    unwrap(PMBR)->PGOInstrGen = PGOGenPath;
+  }
+  if (PGOUsePath) {
+    assert(!PGOGenPath);
+    unwrap(PMBR)->PGOInstrUse = PGOUsePath;
+  }
+#else
+  assert(!PGOGenPath && !PGOUsePath && "Should've caught earlier");
+#endif
 }
 
 // Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo`
@@ -766,6 +785,15 @@ LLVMRustThinLTOAvailable() {
 #endif
 }
 
+extern "C" bool
+LLVMRustPGOAvailable() {
+#ifdef PGO_AVAILABLE
+  return true;
+#else
+  return false;
+#endif
+}
+
 #if LLVM_VERSION_GE(4, 0)
 
 // Here you'll find an implementation of ThinLTO as used by the Rust compiler

src/rustllvm/RustWrapper.cpp

@@ -1021,6 +1021,7 @@ enum class LLVMRustDiagnosticKind {
   OptimizationRemarkAnalysisAliasing,
   OptimizationRemarkOther,
   OptimizationFailure,
+  PGOProfile,
 };
 
 static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) {
@@ -1043,6 +1044,8 @@ static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) {
     return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute;
   case DK_OptimizationRemarkAnalysisAliasing:
     return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing;
+  case DK_PGOProfile:
+    return LLVMRustDiagnosticKind::PGOProfile;
   default:
     return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark)
                ? LLVMRustDiagnosticKind::OptimizationRemarkOther

src/test/run-make-fulldeps/pgo-gen-lto/Makefile

@@ -0,0 +1,8 @@
+-include ../tools.mk
+
+all:
+ifeq ($(PROFILER_SUPPORT),1)
+	$(RUSTC) -Copt-level=3 -Clto=fat -Z pgo-gen="$(TMPDIR)/test.profraw" test.rs
+	$(call RUN,test) || exit 1
+	[ -e "$(TMPDIR)/test.profraw" ] || (echo "No .profraw file"; exit 1)
+endif

src/test/run-make-fulldeps/pgo-gen-lto/test.rs

@@ -0,0 +1,11 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+fn main() {}

src/test/run-make-fulldeps/pgo-gen/Makefile

@@ -0,0 +1,8 @@
+-include ../tools.mk
+
+all:
+ifeq ($(PROFILER_SUPPORT),1)
+	$(RUSTC) -g -Z pgo-gen="$(TMPDIR)/test.profraw" test.rs
+	$(call RUN,test) || exit 1
+	[ -e "$(TMPDIR)/test.profraw" ] || (echo "No .profraw file"; exit 1)
+endif

src/test/run-make-fulldeps/pgo-gen/test.rs

@@ -0,0 +1,11 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+fn main() {}