Merge pull request #693 from bjorn3/abi_refactor

Abi handling refactor

example/example.rs

@@ -202,3 +202,7 @@ fn get_sized_field_ref_from_unsized_type(u: &Unsized) -> &u8 {
 fn get_unsized_field_ref_from_unsized_type(u: &Unsized) -> &str {
     &u.1
 }
+
+pub fn reuse_byref_argument_storage(a: (u8, u16, u32)) -> u8 {
+    a.0
+}

src/abi/comments.rs

@@ -0,0 +1,108 @@
+use std::borrow::Cow;
+
+use rustc::mir;
+
+use crate::prelude::*;
+use crate::abi::pass_mode::*;
+
+pub fn add_args_header_comment(fx: &mut FunctionCx<impl Backend>) {
+    fx.add_global_comment(format!(
+        "kind  loc.idx   param    pass mode                            ty"
+    ));
+}
+
+pub fn add_arg_comment<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
+    kind: &str,
+    local: mir::Local,
+    local_field: Option<usize>,
+    params: EmptySinglePair<Value>,
+    pass_mode: PassMode,
+    ty: Ty<'tcx>,
+) {
+    let local_field = if let Some(local_field) = local_field {
+        Cow::Owned(format!(".{}", local_field))
+    } else {
+        Cow::Borrowed("")
+    };
+    let params = match params {
+        Empty => Cow::Borrowed("-"),
+        Single(param) => Cow::Owned(format!("= {:?}", param)),
+        Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)),
+    };
+    let pass_mode = format!("{:?}", pass_mode);
+    fx.add_global_comment(format!(
+        "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}",
+        kind = kind,
+        local = format!("{:?}", local),
+        local_field = local_field,
+        params = params,
+        pass_mode = pass_mode,
+        ty = ty,
+    ));
+}
+
+pub fn add_locals_header_comment(fx: &mut FunctionCx<impl Backend>) {
+    fx.add_global_comment(String::new());
+    fx.add_global_comment(format!(
+        "kind  local ty                   size  align (abi,pref)"
+    ));
+}
+
+pub fn add_local_place_comments<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
+    place: CPlace<'tcx>,
+    local: Local,
+) {
+    let TyLayout { ty, details } = place.layout();
+    let ty::layout::LayoutDetails {
+        size,
+        align,
+        abi: _,
+        variants: _,
+        fields: _,
+        largest_niche: _,
+    } = details;
+    match *place.inner() {
+        CPlaceInner::Var(var) => {
+            assert_eq!(local, var);
+            fx.add_global_comment(format!(
+                "ssa   {:5} {:20} {:4}b {}, {}",
+                format!("{:?}", local),
+                format!("{:?}", ty),
+                size.bytes(),
+                align.abi.bytes(),
+                align.pref.bytes(),
+            ));
+        }
+        CPlaceInner::Stack(stack_slot) => fx.add_entity_comment(
+            stack_slot,
+            format!(
+                "{:?}: {:?} size={} align={},{}",
+                local,
+                ty,
+                size.bytes(),
+                align.abi.bytes(),
+                align.pref.bytes(),
+            ),
+        ),
+        CPlaceInner::NoPlace => fx.add_global_comment(format!(
+            "zst   {:5} {:20} {:4}b {}, {}",
+            format!("{:?}", local),
+            format!("{:?}", ty),
+            size.bytes(),
+            align.abi.bytes(),
+            align.pref.bytes(),
+        )),
+        CPlaceInner::Addr(addr, None) => fx.add_global_comment(format!(
+            "reuse {:5} {:20} {:4}b {}, {}              storage={}",
+            format!("{:?}", local),
+            format!("{:?}", ty),
+            size.bytes(),
+            align.abi.bytes(),
+            align.pref.bytes(),
+            addr,
+        )),
+        CPlaceInner::Addr(_, Some(_)) => unreachable!(),
+    }
+}

src/abi/mod.rs

@@ -1,152 +1,14 @@
-use std::borrow::Cow;
+#[cfg(debug_assertions)]
+mod comments;
+mod returning;
+mod pass_mode;
 
-use rustc::ty::layout::{FloatTy, Integer, Primitive, Scalar};
 use rustc_target::spec::abi::Abi;
 
 use crate::prelude::*;
+use self::pass_mode::*;
 
-#[derive(Copy, Clone, Debug)]
+pub use self::returning::codegen_return;
-enum PassMode {
-    NoPass,
-    ByVal(Type),
-    ByValPair(Type, Type),
-    ByRef,
-}
-
-#[derive(Copy, Clone, Debug)]
-enum EmptySinglePair<T> {
-    Empty,
-    Single(T),
-    Pair(T, T),
-}
-
-impl<T> EmptySinglePair<T> {
-    fn into_iter(self) -> EmptySinglePairIter<T> {
-        EmptySinglePairIter(self)
-    }
-
-    fn map<U>(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair<U> {
-        match self {
-            Empty => Empty,
-            Single(v) => Single(f(v)),
-            Pair(a, b) => Pair(f(a), f(b)),
-        }
-    }
-}
-
-struct EmptySinglePairIter<T>(EmptySinglePair<T>);
-
-impl<T> Iterator for EmptySinglePairIter<T> {
-    type Item = T;
-
-    fn next(&mut self) -> Option<T> {
-        match std::mem::replace(&mut self.0, Empty) {
-            Empty => None,
-            Single(v) => Some(v),
-            Pair(a, b) => {
-                self.0 = Single(b);
-                Some(a)
-            }
-        }
-    }
-}
-
-impl<T: std::fmt::Debug> EmptySinglePair<T> {
-    fn assert_single(self) -> T {
-        match self {
-            Single(v) => v,
-            _ => panic!("Called assert_single on {:?}", self)
-        }
-    }
-
-    fn assert_pair(self) -> (T, T) {
-        match self {
-            Pair(a, b) => (a, b),
-            _ => panic!("Called assert_pair on {:?}", self)
-        }
-    }
-}
-
-use EmptySinglePair::*;
-
-impl PassMode {
-    fn get_param_ty(self, fx: &FunctionCx<impl Backend>) -> EmptySinglePair<Type> {
-        match self {
-            PassMode::NoPass => Empty,
-            PassMode::ByVal(clif_type) => Single(clif_type),
-            PassMode::ByValPair(a, b) => Pair(a, b),
-            PassMode::ByRef => Single(fx.pointer_type),
-        }
-    }
-}
-
-pub fn scalar_to_clif_type(tcx: TyCtxt, scalar: Scalar) -> Type {
-    match scalar.value {
-        Primitive::Int(int, _sign) => match int {
-            Integer::I8 => types::I8,
-            Integer::I16 => types::I16,
-            Integer::I32 => types::I32,
-            Integer::I64 => types::I64,
-            Integer::I128 => types::I128,
-        },
-        Primitive::Float(flt) => match flt {
-            FloatTy::F32 => types::F32,
-            FloatTy::F64 => types::F64,
-        },
-        Primitive::Pointer => pointer_ty(tcx),
-    }
-}
-
-fn get_pass_mode<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    layout: TyLayout<'tcx>,
-) -> PassMode {
-    assert!(!layout.is_unsized());
-
-    if layout.is_zst() {
-        // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer
-        PassMode::NoPass
-    } else {
-        match &layout.abi {
-            layout::Abi::Uninhabited => PassMode::NoPass,
-            layout::Abi::Scalar(scalar) => {
-                PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone()))
-            }
-            layout::Abi::ScalarPair(a, b) => {
-                let a = scalar_to_clif_type(tcx, a.clone());
-                let b = scalar_to_clif_type(tcx, b.clone());
-                if a == types::I128 && b == types::I128 {
-                    // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are
-                    // available on x86_64. Cranelift gets confused when too many return params
-                    // are used.
-                    PassMode::ByRef
-                } else {
-                    PassMode::ByValPair(a, b)
-                }
-            }
-
-            // FIXME implement Vector Abi in a cg_llvm compatible way
-            layout::Abi::Vector { .. } => PassMode::ByRef,
-
-            layout::Abi::Aggregate { .. } => PassMode::ByRef,
-        }
-    }
-}
-
-fn adjust_arg_for_abi<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
-    arg: CValue<'tcx>,
-) -> EmptySinglePair<Value> {
-    match get_pass_mode(fx.tcx, arg.layout()) {
-        PassMode::NoPass => Empty,
-        PassMode::ByVal(_) => Single(arg.load_scalar(fx)),
-        PassMode::ByValPair(_, _) => {
-            let (a, b) = arg.load_scalar_pair(fx);
-            Pair(a, b)
-        }
-        PassMode::ByRef => Single(arg.force_stack(fx)),
-    }
-}
 
 fn clif_sig_from_fn_sig<'tcx>(tcx: TyCtxt<'tcx>, sig: FnSig<'tcx>, is_vtable_fn: bool) -> Signature {
     let abi = match sig.abi {
@@ -187,12 +49,7 @@ fn clif_sig_from_fn_sig<'tcx>(tcx: TyCtxt<'tcx>, sig: FnSig<'tcx>, is_vtable_fn:
                 // See https://github.com/rust-lang/rust/blob/37b6a5e5e82497caf5353d9d856e4eb5d14cbe06/src/librustc/ty/layout.rs#L2519-L2572 for more info
                 layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))).unwrap();
             }
-            match get_pass_mode(tcx, layout) {
+            get_pass_mode(tcx, layout).get_param_ty(tcx).into_iter()
-                PassMode::NoPass => Empty,
-                PassMode::ByVal(clif_ty) => Single(clif_ty),
-                PassMode::ByValPair(clif_ty_a, clif_ty_b) => Pair(clif_ty_a, clif_ty_b),
-                PassMode::ByRef => Single(pointer_ty(tcx)),
-            }.into_iter()
         }).flatten();
 
     let (params, returns) = match get_pass_mode(tcx, tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap()) {
@@ -336,47 +193,6 @@ impl<'tcx, B: Backend + 'static> FunctionCx<'_, 'tcx, B> {
     }
 }
 
-#[cfg(debug_assertions)]
-fn add_arg_comment<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
-    msg: &str,
-    local: mir::Local,
-    local_field: Option<usize>,
-    params: EmptySinglePair<Value>,
-    pass_mode: PassMode,
-    ssa: crate::analyze::Flags,
-    ty: Ty<'tcx>,
-) {
-    let local_field = if let Some(local_field) = local_field {
-        Cow::Owned(format!(".{}", local_field))
-    } else {
-        Cow::Borrowed("")
-    };
-    let params = match params {
-        Empty => Cow::Borrowed("-"),
-        Single(param) => Cow::Owned(format!("= {:?}", param)),
-        Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)),
-    };
-    let pass_mode = format!("{:?}", pass_mode);
-    fx.add_global_comment(format!(
-        "{msg:5} {local:>3}{local_field:<5} {params:10} {pass_mode:36} {ssa:10} {ty:?}",
-        msg = msg,
-        local = format!("{:?}", local),
-        local_field = local_field,
-        params = params,
-        pass_mode = pass_mode,
-        ssa = format!("{:?}", ssa),
-        ty = ty,
-    ));
-}
-
-#[cfg(debug_assertions)]
-fn add_local_header_comment(fx: &mut FunctionCx<impl Backend>) {
-    fx.add_global_comment(format!(
-        "msg   loc.idx    param    pass mode                            ssa flags  ty"
-    ));
-}
-
 fn local_place<'tcx>(
     fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
     local: Local,
@@ -386,92 +202,17 @@ fn local_place<'tcx>(
     let place = if is_ssa {
         CPlace::new_var(fx, local, layout)
     } else {
-        let place = CPlace::new_stack_slot(fx, layout.ty);
+        CPlace::new_stack_slot(fx, layout.ty)
+    };
 
     #[cfg(debug_assertions)]
-        {
+    self::comments::add_local_place_comments(fx, place, local);
-            let TyLayout { ty, details } = layout;
-            let ty::layout::LayoutDetails {
-                size,
-                align,
-                abi: _,
-                variants: _,
-                fields: _,
-                largest_niche: _,
-            } = details;
-            match *place.inner() {
-                CPlaceInner::Stack(stack_slot) => fx.add_entity_comment(
-                    stack_slot,
-                    format!(
-                        "{:?}: {:?} size={} align={},{}",
-                        local,
-                        ty,
-                        size.bytes(),
-                        align.abi.bytes(),
-                        align.pref.bytes(),
-                    ),
-                ),
-                CPlaceInner::NoPlace => fx.add_global_comment(format!(
-                    "zst    {:?}: {:?} size={} align={}, {}",
-                    local,
-                    ty,
-                    size.bytes(),
-                    align.abi.bytes(),
-                    align.pref.bytes(),
-                )),
-                _ => unreachable!(),
-            }
-        }
-
-        place
-    };
 
     let prev_place = fx.local_map.insert(local, place);
     debug_assert!(prev_place.is_none());
     fx.local_map[&local]
 }
 
-fn cvalue_for_param<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
-    start_ebb: Ebb,
-    local: mir::Local,
-    local_field: Option<usize>,
-    arg_ty: Ty<'tcx>,
-    ssa_flags: crate::analyze::Flags,
-) -> Option<CValue<'tcx>> {
-    let layout = fx.layout_of(arg_ty);
-    let pass_mode = get_pass_mode(fx.tcx, fx.layout_of(arg_ty));
-
-    if let PassMode::NoPass = pass_mode {
-        return None;
-    }
-
-    let clif_types = pass_mode.get_param_ty(fx);
-    let ebb_params = clif_types.map(|t| fx.bcx.append_ebb_param(start_ebb, t));
-
-    #[cfg(debug_assertions)]
-    add_arg_comment(
-        fx,
-        "arg",
-        local,
-        local_field,
-        ebb_params,
-        pass_mode,
-        ssa_flags,
-        arg_ty,
-    );
-
-    match pass_mode {
-        PassMode::NoPass => unreachable!(),
-        PassMode::ByVal(_) => Some(CValue::by_val(ebb_params.assert_single(), layout)),
-        PassMode::ByValPair(_, _) => {
-            let (a, b) = ebb_params.assert_pair();
-            Some(CValue::by_val_pair(a, b, layout))
-        }
-        PassMode::ByRef => Some(CValue::by_ref(ebb_params.assert_single(), layout)),
-    }
-}
-
 pub fn codegen_fn_prelude(
     fx: &mut FunctionCx<'_, '_, impl Backend>,
     start_ebb: Ebb,
@@ -479,33 +220,9 @@ pub fn codegen_fn_prelude(
     let ssa_analyzed = crate::analyze::analyze(fx);
 
     #[cfg(debug_assertions)]
-    fx.add_global_comment(format!("ssa {:?}", ssa_analyzed));
+    self::comments::add_args_header_comment(fx);
 
-    let ret_layout = fx.return_layout();
+    self::returning::codegen_return_param(fx, &ssa_analyzed, start_ebb);
-    let output_pass_mode = get_pass_mode(fx.tcx, fx.return_layout());
-    let ret_param = match output_pass_mode {
-        PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None,
-        PassMode::ByRef => Some(fx.bcx.append_ebb_param(start_ebb, fx.pointer_type)),
-    };
-
-    #[cfg(debug_assertions)]
-    {
-        add_local_header_comment(fx);
-        let ret_param = match ret_param {
-            Some(param) => Single(param),
-            None => Empty,
-        };
-        add_arg_comment(
-            fx,
-            "ret",
-            RETURN_PLACE,
-            None,
-            ret_param,
-            output_pass_mode,
-            ssa_analyzed[&RETURN_PLACE],
-            ret_layout.ty,
-        );
-    }
 
     // None means pass_mode == NoPass
     enum ArgKind<'tcx> {
@@ -539,7 +256,6 @@ pub fn codegen_fn_prelude(
                         local,
                         Some(i),
                         arg_ty,
-                        ssa_analyzed[&local],
                     );
                     params.push(param);
                 }
@@ -547,7 +263,7 @@ pub fn codegen_fn_prelude(
                 (local, ArgKind::Spread(params), arg_ty)
             } else {
                 let param =
-                    cvalue_for_param(fx, start_ebb, local, None, arg_ty, ssa_analyzed[&local]);
+                    cvalue_for_param(fx, start_ebb, local, None, arg_ty);
                 (local, ArgKind::Normal(param), arg_ty)
             }
         })
@@ -555,26 +271,8 @@ pub fn codegen_fn_prelude(
 
     fx.bcx.switch_to_block(start_ebb);
 
-    match output_pass_mode {
+    #[cfg(debug_assertions)]
-        PassMode::NoPass => {
+    self::comments::add_locals_header_comment(fx);
-            fx.local_map
-                .insert(RETURN_PLACE, CPlace::no_place(ret_layout));
-        }
-        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => {
-            let is_ssa = !ssa_analyzed
-                .get(&RETURN_PLACE)
-                .unwrap()
-                .contains(crate::analyze::Flags::NOT_SSA);
-
-            local_place(fx, RETURN_PLACE, ret_layout, is_ssa);
-        }
-        PassMode::ByRef => {
-            fx.local_map.insert(
-                RETURN_PLACE,
-                CPlace::for_addr(ret_param.unwrap(), ret_layout),
-            );
-        }
-    }
 
     for (local, arg_kind, ty) in func_params {
         let layout = fx.layout_of(ty);
@@ -584,6 +282,34 @@ pub fn codegen_fn_prelude(
             .unwrap()
             .contains(crate::analyze::Flags::NOT_SSA);
 
+        match arg_kind {
+            ArgKind::Normal(Some(val)) => {
+                if let Some(addr) = val.try_to_addr() {
+                    let local_decl = &fx.mir.local_decls[local];
+                    //                             v this ! is important
+                    let internally_mutable = !val.layout().ty.is_freeze(
+                        fx.tcx,
+                        ParamEnv::reveal_all(),
+                        local_decl.source_info.span,
+                    );
+                    if local_decl.mutability == mir::Mutability::Not && internally_mutable {
+                        // We wont mutate this argument, so it is fine to borrow the backing storage
+                        // of this argument, to prevent a copy.
+
+                        let place = CPlace::for_addr(addr, val.layout());
+
+                        #[cfg(debug_assertions)]
+                        self::comments::add_local_place_comments(fx, place, local);
+
+                        let prev_place = fx.local_map.insert(local, place);
+                        debug_assert!(prev_place.is_none());
+                        continue;
+                    }
+                }
+            }
+            _ => {}
+        }
+
         let place = local_place(fx, local, layout, is_ssa);
 
         match arg_kind {
@@ -706,18 +432,6 @@ fn codegen_call_inner<'tcx>(
 ) {
     let fn_sig = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), &fn_ty.fn_sig(fx.tcx));
 
-    let ret_layout = fx.layout_of(fn_sig.output());
-
-    let output_pass_mode = get_pass_mode(fx.tcx, fx.layout_of(fn_sig.output()));
-    let return_ptr = match output_pass_mode {
-        PassMode::NoPass => None,
-        PassMode::ByRef => match ret_place {
-            Some(ret_place) => Some(ret_place.to_addr(fx)),
-            None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)),
-        },
-        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None,
-    };
-
     let instance = match fn_ty.sty {
         ty::FnDef(def_id, substs) => {
             Some(Instance::resolve(fx.tcx, ParamEnv::reveal_all(), def_id, substs).unwrap())
@@ -766,6 +480,7 @@ fn codegen_call_inner<'tcx>(
         }
     };
 
+    let (call_inst, call_args) = self::returning::codegen_with_call_return_arg(fx, fn_sig, ret_place, |fx, return_ptr| {
         let call_args: Vec<Value> = return_ptr
             .into_iter()
             .chain(first_arg.into_iter())
@@ -787,6 +502,9 @@ fn codegen_call_inner<'tcx>(
             fx.bcx.ins().call(func_ref, &call_args)
         };
 
+        (call_inst, call_args)
+    });
+
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
         if fn_sig.abi != Abi::C {
@@ -806,24 +524,6 @@ fn codegen_call_inner<'tcx>(
             .collect::<Vec<AbiParam>>();
         fx.bcx.func.dfg.signatures[sig_ref].params = abi_params;
     }
-
-    match output_pass_mode {
-        PassMode::NoPass => {}
-        PassMode::ByVal(_) => {
-            if let Some(ret_place) = ret_place {
-                let ret_val = fx.bcx.inst_results(call_inst)[0];
-                ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout));
-            }
-        }
-        PassMode::ByValPair(_, _) => {
-            if let Some(ret_place) = ret_place {
-                let ret_val_a = fx.bcx.inst_results(call_inst)[0];
-                let ret_val_b = fx.bcx.inst_results(call_inst)[1];
-                ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout));
-            }
-        }
-        PassMode::ByRef => {}
-    }
 }
 
 pub fn codegen_drop<'tcx>(
@@ -875,21 +575,3 @@ pub fn codegen_drop<'tcx>(
         }
     }
 }
-
-pub fn codegen_return(fx: &mut FunctionCx<impl Backend>) {
-    match get_pass_mode(fx.tcx, fx.return_layout()) {
-        PassMode::NoPass | PassMode::ByRef => {
-            fx.bcx.ins().return_(&[]);
-        }
-        PassMode::ByVal(_) => {
-            let place = fx.get_local_place(RETURN_PLACE);
-            let ret_val = place.to_cvalue(fx).load_scalar(fx);
-            fx.bcx.ins().return_(&[ret_val]);
-        }
-        PassMode::ByValPair(_, _) => {
-            let place = fx.get_local_place(RETURN_PLACE);
-            let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
-            fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);
-        }
-    }
-}

src/abi/pass_mode.rs

@@ -0,0 +1,166 @@
+use crate::prelude::*;
+
+#[derive(Copy, Clone, Debug)]
+pub enum PassMode {
+    NoPass,
+    ByVal(Type),
+    ByValPair(Type, Type),
+    ByRef,
+}
+
+#[derive(Copy, Clone, Debug)]
+pub enum EmptySinglePair<T> {
+    Empty,
+    Single(T),
+    Pair(T, T),
+}
+
+impl<T> EmptySinglePair<T> {
+    pub fn into_iter(self) -> EmptySinglePairIter<T> {
+        EmptySinglePairIter(self)
+    }
+
+    pub fn map<U>(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair<U> {
+        match self {
+            Empty => Empty,
+            Single(v) => Single(f(v)),
+            Pair(a, b) => Pair(f(a), f(b)),
+        }
+    }
+}
+
+pub struct EmptySinglePairIter<T>(EmptySinglePair<T>);
+
+impl<T> Iterator for EmptySinglePairIter<T> {
+    type Item = T;
+
+    fn next(&mut self) -> Option<T> {
+        match std::mem::replace(&mut self.0, Empty) {
+            Empty => None,
+            Single(v) => Some(v),
+            Pair(a, b) => {
+                self.0 = Single(b);
+                Some(a)
+            }
+        }
+    }
+}
+
+impl<T: std::fmt::Debug> EmptySinglePair<T> {
+    pub fn assert_single(self) -> T {
+        match self {
+            Single(v) => v,
+            _ => panic!("Called assert_single on {:?}", self)
+        }
+    }
+
+    pub fn assert_pair(self) -> (T, T) {
+        match self {
+            Pair(a, b) => (a, b),
+            _ => panic!("Called assert_pair on {:?}", self)
+        }
+    }
+}
+
+pub use EmptySinglePair::*;
+
+impl PassMode {
+    pub fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair<Type> {
+        match self {
+            PassMode::NoPass => Empty,
+            PassMode::ByVal(clif_type) => Single(clif_type),
+            PassMode::ByValPair(a, b) => Pair(a, b),
+            PassMode::ByRef => Single(pointer_ty(tcx)),
+        }
+    }
+}
+
+pub fn get_pass_mode<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    layout: TyLayout<'tcx>,
+) -> PassMode {
+    assert!(!layout.is_unsized());
+
+    if layout.is_zst() {
+        // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer
+        PassMode::NoPass
+    } else {
+        match &layout.abi {
+            layout::Abi::Uninhabited => PassMode::NoPass,
+            layout::Abi::Scalar(scalar) => {
+                PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone()))
+            }
+            layout::Abi::ScalarPair(a, b) => {
+                let a = scalar_to_clif_type(tcx, a.clone());
+                let b = scalar_to_clif_type(tcx, b.clone());
+                if a == types::I128 && b == types::I128 {
+                    // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are
+                    // available on x86_64. Cranelift gets confused when too many return params
+                    // are used.
+                    PassMode::ByRef
+                } else {
+                    PassMode::ByValPair(a, b)
+                }
+            }
+
+            // FIXME implement Vector Abi in a cg_llvm compatible way
+            layout::Abi::Vector { .. } => PassMode::ByRef,
+
+            layout::Abi::Aggregate { .. } => PassMode::ByRef,
+        }
+    }
+}
+
+pub fn adjust_arg_for_abi<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
+    arg: CValue<'tcx>,
+) -> EmptySinglePair<Value> {
+    match get_pass_mode(fx.tcx, arg.layout()) {
+        PassMode::NoPass => Empty,
+        PassMode::ByVal(_) => Single(arg.load_scalar(fx)),
+        PassMode::ByValPair(_, _) => {
+            let (a, b) = arg.load_scalar_pair(fx);
+            Pair(a, b)
+        }
+        PassMode::ByRef => Single(arg.force_stack(fx)),
+    }
+}
+
+pub fn cvalue_for_param<'tcx>(
+    fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
+    start_ebb: Ebb,
+    local: mir::Local,
+    local_field: Option<usize>,
+    arg_ty: Ty<'tcx>,
+) -> Option<CValue<'tcx>> {
+    let layout = fx.layout_of(arg_ty);
+    let pass_mode = get_pass_mode(fx.tcx, fx.layout_of(arg_ty));
+
+    if let PassMode::NoPass = pass_mode {
+        return None;
+    }
+
+    let clif_types = pass_mode.get_param_ty(fx.tcx);
+    let ebb_params = clif_types.map(|t| fx.bcx.append_ebb_param(start_ebb, t));
+
+    #[cfg(debug_assertions)]
+    crate::abi::comments::add_arg_comment(
+        fx,
+        "arg",
+        local,
+        local_field,
+        ebb_params,
+        pass_mode,
+        arg_ty,
+    );
+
+    match pass_mode {
+        PassMode::NoPass => unreachable!(),
+        PassMode::ByVal(_) => Some(CValue::by_val(ebb_params.assert_single(), layout)),
+        PassMode::ByValPair(_, _) => {
+            let (a, b) = ebb_params.assert_pair();
+            Some(CValue::by_val_pair(a, b, layout))
+        }
+        PassMode::ByRef => Some(CValue::by_ref(ebb_params.assert_single(), layout)),
+    }
+}

src/abi/returning.rs

@@ -0,0 +1,108 @@
+use crate::prelude::*;
+use crate::abi::pass_mode::*;
+
+pub fn codegen_return_param(
+    fx: &mut FunctionCx<impl Backend>,
+    ssa_analyzed: &HashMap<Local, crate::analyze::Flags>,
+    start_ebb: Ebb,
+) {
+    let ret_layout = fx.return_layout();
+    let output_pass_mode = get_pass_mode(fx.tcx, fx.return_layout());
+
+    let ret_param = match output_pass_mode {
+        PassMode::NoPass => {
+            fx.local_map
+                .insert(RETURN_PLACE, CPlace::no_place(ret_layout));
+            Empty
+        }
+        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => {
+            let is_ssa = !ssa_analyzed
+                .get(&RETURN_PLACE)
+                .unwrap()
+                .contains(crate::analyze::Flags::NOT_SSA);
+
+            super::local_place(fx, RETURN_PLACE, ret_layout, is_ssa);
+
+            Empty
+        }
+        PassMode::ByRef => {
+            let ret_param = fx.bcx.append_ebb_param(start_ebb, fx.pointer_type);
+            fx.local_map.insert(
+                RETURN_PLACE,
+                CPlace::for_addr(ret_param, ret_layout),
+            );
+
+            Single(ret_param)
+        }
+    };
+
+    #[cfg(debug_assertions)]
+    crate::abi::comments::add_arg_comment(
+        fx,
+        "ret",
+        RETURN_PLACE,
+        None,
+        ret_param,
+        output_pass_mode,
+        ret_layout.ty,
+    );
+}
+
+pub fn codegen_with_call_return_arg<'tcx, B: Backend, T>(
+    fx: &mut FunctionCx<'_, 'tcx, B>,
+    fn_sig: FnSig<'tcx>,
+    ret_place: Option<CPlace<'tcx>>,
+    f: impl FnOnce(&mut FunctionCx<'_, 'tcx, B>, Option<Value>) -> (Inst, T),
+) -> (Inst, T) {
+    let ret_layout = fx.layout_of(fn_sig.output());
+
+    let output_pass_mode = get_pass_mode(fx.tcx, ret_layout);
+    let return_ptr = match output_pass_mode {
+        PassMode::NoPass => None,
+        PassMode::ByRef => match ret_place {
+            Some(ret_place) => Some(ret_place.to_addr(fx)),
+            None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)),
+        },
+        PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None,
+    };
+
+    let (call_inst, meta) = f(fx, return_ptr);
+
+    match output_pass_mode {
+        PassMode::NoPass => {}
+        PassMode::ByVal(_) => {
+            if let Some(ret_place) = ret_place {
+                let ret_val = fx.bcx.inst_results(call_inst)[0];
+                ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout));
+            }
+        }
+        PassMode::ByValPair(_, _) => {
+            if let Some(ret_place) = ret_place {
+                let ret_val_a = fx.bcx.inst_results(call_inst)[0];
+                let ret_val_b = fx.bcx.inst_results(call_inst)[1];
+                ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout));
+            }
+        }
+        PassMode::ByRef => {}
+    }
+
+    (call_inst, meta)
+}
+
+pub fn codegen_return(fx: &mut FunctionCx<impl Backend>) {
+    match get_pass_mode(fx.tcx, fx.return_layout()) {
+        PassMode::NoPass | PassMode::ByRef => {
+            fx.bcx.ins().return_(&[]);
+        }
+        PassMode::ByVal(_) => {
+            let place = fx.get_local_place(RETURN_PLACE);
+            let ret_val = place.to_cvalue(fx).load_scalar(fx);
+            fx.bcx.ins().return_(&[ret_val]);
+        }
+        PassMode::ByValPair(_, _) => {
+            let place = fx.get_local_place(RETURN_PLACE);
+            let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
+            fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);
+        }
+    }
+}

src/common.rs

@@ -1,3 +1,4 @@
+use rustc::ty::layout::{FloatTy, Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
 use cranelift::codegen::ir::{Opcode, InstructionData, ValueDef};
@@ -17,6 +18,23 @@ pub fn pointer_ty(tcx: TyCtxt) -> types::Type {
     }
 }
 
+pub fn scalar_to_clif_type(tcx: TyCtxt, scalar: Scalar) -> Type {
+    match scalar.value {
+        Primitive::Int(int, _sign) => match int {
+            Integer::I8 => types::I8,
+            Integer::I16 => types::I16,
+            Integer::I32 => types::I32,
+            Integer::I64 => types::I64,
+            Integer::I128 => types::I128,
+        },
+        Primitive::Float(flt) => match flt {
+            FloatTy::F32 => types::F32,
+            FloatTy::F64 => types::F64,
+        },
+        Primitive::Pointer => pointer_ty(tcx),
+    }
+}
+
 pub fn clif_type_from_ty<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,

src/value_and_place.rs

@@ -63,6 +63,13 @@ impl<'tcx> CValue<'tcx> {
         }
     }
 
+    pub fn try_to_addr(self) -> Option<Value> {
+        match self.0 {
+            CValueInner::ByRef(addr) => Some(addr),
+            CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None,
+        }
+    }
+
     /// Load a value with layout.abi of scalar
     pub fn load_scalar<'a>(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Value {
         let layout = self.1;