251 lines
9.7 KiB
Rust
251 lines
9.7 KiB
Rust
// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
|
|
// file at the top-level directory of this distribution and at
|
|
// http://rust-lang.org/COPYRIGHT.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
|
|
#![crate_name = "alloc_jemalloc"]
|
|
#![crate_type = "rlib"]
|
|
#![no_std]
|
|
#![unstable(feature = "alloc_jemalloc",
|
|
reason = "this library is unlikely to be stabilized in its current \
|
|
form or name",
|
|
issue = "27783")]
|
|
#![deny(warnings)]
|
|
#![feature(alloc)]
|
|
#![feature(alloc_system)]
|
|
#![feature(libc)]
|
|
#![feature(linkage)]
|
|
#![feature(staged_api)]
|
|
#![cfg_attr(dummy_jemalloc, allow(dead_code))]
|
|
#![cfg_attr(not(dummy_jemalloc), feature(allocator_api))]
|
|
|
|
extern crate alloc;
|
|
extern crate alloc_system;
|
|
extern crate libc;
|
|
|
|
#[cfg(not(dummy_jemalloc))]
|
|
pub use contents::*;
|
|
#[cfg(not(dummy_jemalloc))]
|
|
mod contents {
|
|
use core::ptr;
|
|
|
|
use alloc::heap::{Alloc, AllocErr, Layout};
|
|
use alloc_system::System;
|
|
use libc::{c_int, c_void, size_t};
|
|
|
|
// Note that the symbols here are prefixed by default on macOS and Windows (we
|
|
// don't explicitly request it), and on Android and DragonFly we explicitly
|
|
// request it as unprefixing cause segfaults (mismatches in allocators).
|
|
extern "C" {
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_mallocx")]
|
|
fn mallocx(size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_calloc")]
|
|
fn calloc(size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_rallocx")]
|
|
fn rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_xallocx")]
|
|
fn xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t;
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_sdallocx")]
|
|
fn sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
|
|
#[cfg_attr(any(target_os = "macos", target_os = "android", target_os = "ios",
|
|
target_os = "dragonfly", target_os = "windows", target_env = "musl"),
|
|
link_name = "je_nallocx")]
|
|
fn nallocx(size: size_t, flags: c_int) -> size_t;
|
|
}
|
|
|
|
const MALLOCX_ZERO: c_int = 0x40;
|
|
|
|
// The minimum alignment guaranteed by the architecture. This value is used to
|
|
// add fast paths for low alignment values. In practice, the alignment is a
|
|
// constant at the call site and the branch will be optimized out.
|
|
#[cfg(all(any(target_arch = "arm",
|
|
target_arch = "mips",
|
|
target_arch = "powerpc")))]
|
|
const MIN_ALIGN: usize = 8;
|
|
#[cfg(all(any(target_arch = "x86",
|
|
target_arch = "x86_64",
|
|
target_arch = "aarch64",
|
|
target_arch = "powerpc64",
|
|
target_arch = "mips64",
|
|
target_arch = "s390x",
|
|
target_arch = "sparc64")))]
|
|
const MIN_ALIGN: usize = 16;
|
|
|
|
// MALLOCX_ALIGN(a) macro
|
|
fn mallocx_align(a: usize) -> c_int {
|
|
a.trailing_zeros() as c_int
|
|
}
|
|
|
|
fn align_to_flags(align: usize) -> c_int {
|
|
if align <= MIN_ALIGN {
|
|
0
|
|
} else {
|
|
mallocx_align(align)
|
|
}
|
|
}
|
|
|
|
// for symbol names src/librustc/middle/allocator.rs
|
|
// for signatures src/librustc_allocator/lib.rs
|
|
|
|
// linkage directives are provided as part of the current compiler allocator
|
|
// ABI
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc(size: usize,
|
|
align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let flags = align_to_flags(align);
|
|
let ptr = mallocx(size as size_t, flags) as *mut u8;
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(size, align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_oom(err: *const u8) -> ! {
|
|
System.oom((*(err as *const AllocErr)).clone())
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_dealloc(ptr: *mut u8,
|
|
size: usize,
|
|
align: usize) {
|
|
let flags = align_to_flags(align);
|
|
sdallocx(ptr as *mut c_void, size, flags);
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_usable_size(layout: *const u8,
|
|
min: *mut usize,
|
|
max: *mut usize) {
|
|
let layout = &*(layout as *const Layout);
|
|
let flags = align_to_flags(layout.align());
|
|
let size = nallocx(layout.size(), flags) as usize;
|
|
*min = layout.size();
|
|
if size > 0 {
|
|
*max = size;
|
|
} else {
|
|
*max = layout.size();
|
|
}
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_realloc(ptr: *mut u8,
|
|
_old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
if new_align != old_align {
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Unsupported { details: "can't change alignments" });
|
|
return 0 as *mut u8
|
|
}
|
|
|
|
let flags = align_to_flags(new_align);
|
|
let ptr = rallocx(ptr as *mut c_void, new_size, flags) as *mut u8;
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(new_size, new_align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc_zeroed(size: usize,
|
|
align: usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let ptr = if align <= MIN_ALIGN {
|
|
calloc(size as size_t, 1) as *mut u8
|
|
} else {
|
|
let flags = align_to_flags(align) | MALLOCX_ZERO;
|
|
mallocx(size as size_t, flags) as *mut u8
|
|
};
|
|
if ptr.is_null() {
|
|
let layout = Layout::from_size_align_unchecked(size, align);
|
|
ptr::write(err as *mut AllocErr,
|
|
AllocErr::Exhausted { request: layout });
|
|
}
|
|
ptr
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_alloc_excess(size: usize,
|
|
align: usize,
|
|
excess: *mut usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let p = __rde_alloc(size, align, err);
|
|
if !p.is_null() {
|
|
*excess = size;
|
|
}
|
|
return p
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_realloc_excess(ptr: *mut u8,
|
|
old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize,
|
|
excess: *mut usize,
|
|
err: *mut u8) -> *mut u8 {
|
|
let p = __rde_realloc(ptr, old_size, old_align, new_size, new_align, err);
|
|
if !p.is_null() {
|
|
*excess = new_size;
|
|
}
|
|
return p
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_grow_in_place(ptr: *mut u8,
|
|
old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize) -> u8 {
|
|
__rde_shrink_in_place(ptr, old_size, old_align, new_size, new_align)
|
|
}
|
|
|
|
#[no_mangle]
|
|
#[linkage = "external"]
|
|
pub unsafe extern fn __rde_shrink_in_place(ptr: *mut u8,
|
|
_old_size: usize,
|
|
old_align: usize,
|
|
new_size: usize,
|
|
new_align: usize) -> u8 {
|
|
if old_align == new_align {
|
|
let flags = align_to_flags(new_align);
|
|
(xallocx(ptr as *mut c_void, new_size, 0, flags) == new_size) as u8
|
|
} else {
|
|
0
|
|
}
|
|
}
|
|
}
|