383 lines
13 KiB
C++
383 lines
13 KiB
C++
//===-- sanitizer_allocator_primary32.h -------------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of the Sanitizer Allocator.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
#ifndef SANITIZER_ALLOCATOR_H
|
|
#error This file must be included inside sanitizer_allocator.h
|
|
#endif
|
|
|
|
template<class SizeClassAllocator> struct SizeClassAllocator32LocalCache;
|
|
|
|
// SizeClassAllocator32 -- allocator for 32-bit address space.
|
|
// This allocator can theoretically be used on 64-bit arch, but there it is less
|
|
// efficient than SizeClassAllocator64.
|
|
//
|
|
// [kSpaceBeg, kSpaceBeg + kSpaceSize) is the range of addresses which can
|
|
// be returned by MmapOrDie().
|
|
//
|
|
// Region:
|
|
// a result of a single call to MmapAlignedOrDieOnFatalError(kRegionSize,
|
|
// kRegionSize).
|
|
// Since the regions are aligned by kRegionSize, there are exactly
|
|
// kNumPossibleRegions possible regions in the address space and so we keep
|
|
// a ByteMap possible_regions to store the size classes of each Region.
|
|
// 0 size class means the region is not used by the allocator.
|
|
//
|
|
// One Region is used to allocate chunks of a single size class.
|
|
// A Region looks like this:
|
|
// UserChunk1 .. UserChunkN <gap> MetaChunkN .. MetaChunk1
|
|
//
|
|
// In order to avoid false sharing the objects of this class should be
|
|
// chache-line aligned.
|
|
|
|
struct SizeClassAllocator32FlagMasks { // Bit masks.
|
|
enum {
|
|
kRandomShuffleChunks = 1,
|
|
kUseSeparateSizeClassForBatch = 2,
|
|
};
|
|
};
|
|
|
|
template <class Params>
|
|
class SizeClassAllocator32 {
|
|
private:
|
|
static const u64 kTwoLevelByteMapSize1 =
|
|
(Params::kSpaceSize >> Params::kRegionSizeLog) >> 12;
|
|
static const u64 kMinFirstMapSizeTwoLevelByteMap = 4;
|
|
|
|
public:
|
|
using AddressSpaceView = typename Params::AddressSpaceView;
|
|
static const uptr kSpaceBeg = Params::kSpaceBeg;
|
|
static const u64 kSpaceSize = Params::kSpaceSize;
|
|
static const uptr kMetadataSize = Params::kMetadataSize;
|
|
typedef typename Params::SizeClassMap SizeClassMap;
|
|
static const uptr kRegionSizeLog = Params::kRegionSizeLog;
|
|
typedef typename Params::MapUnmapCallback MapUnmapCallback;
|
|
using ByteMap = typename conditional<
|
|
(kTwoLevelByteMapSize1 < kMinFirstMapSizeTwoLevelByteMap),
|
|
FlatByteMap<(Params::kSpaceSize >> Params::kRegionSizeLog),
|
|
AddressSpaceView>,
|
|
TwoLevelByteMap<kTwoLevelByteMapSize1, 1 << 12, AddressSpaceView>>::type;
|
|
|
|
COMPILER_CHECK(!SANITIZER_SIGN_EXTENDED_ADDRESSES ||
|
|
(kSpaceSize & (kSpaceSize - 1)) == 0);
|
|
|
|
static const bool kRandomShuffleChunks = Params::kFlags &
|
|
SizeClassAllocator32FlagMasks::kRandomShuffleChunks;
|
|
static const bool kUseSeparateSizeClassForBatch = Params::kFlags &
|
|
SizeClassAllocator32FlagMasks::kUseSeparateSizeClassForBatch;
|
|
|
|
struct TransferBatch {
|
|
static const uptr kMaxNumCached = SizeClassMap::kMaxNumCachedHint - 2;
|
|
void SetFromArray(void *batch[], uptr count) {
|
|
DCHECK_LE(count, kMaxNumCached);
|
|
count_ = count;
|
|
for (uptr i = 0; i < count; i++)
|
|
batch_[i] = batch[i];
|
|
}
|
|
uptr Count() const { return count_; }
|
|
void Clear() { count_ = 0; }
|
|
void Add(void *ptr) {
|
|
batch_[count_++] = ptr;
|
|
DCHECK_LE(count_, kMaxNumCached);
|
|
}
|
|
void CopyToArray(void *to_batch[]) const {
|
|
for (uptr i = 0, n = Count(); i < n; i++)
|
|
to_batch[i] = batch_[i];
|
|
}
|
|
|
|
// How much memory do we need for a batch containing n elements.
|
|
static uptr AllocationSizeRequiredForNElements(uptr n) {
|
|
return sizeof(uptr) * 2 + sizeof(void *) * n;
|
|
}
|
|
static uptr MaxCached(uptr size) {
|
|
return Min(kMaxNumCached, SizeClassMap::MaxCachedHint(size));
|
|
}
|
|
|
|
TransferBatch *next;
|
|
|
|
private:
|
|
uptr count_;
|
|
void *batch_[kMaxNumCached];
|
|
};
|
|
|
|
static const uptr kBatchSize = sizeof(TransferBatch);
|
|
COMPILER_CHECK((kBatchSize & (kBatchSize - 1)) == 0);
|
|
COMPILER_CHECK(kBatchSize == SizeClassMap::kMaxNumCachedHint * sizeof(uptr));
|
|
|
|
static uptr ClassIdToSize(uptr class_id) {
|
|
return (class_id == SizeClassMap::kBatchClassID) ?
|
|
kBatchSize : SizeClassMap::Size(class_id);
|
|
}
|
|
|
|
typedef SizeClassAllocator32<Params> ThisT;
|
|
typedef SizeClassAllocator32LocalCache<ThisT> AllocatorCache;
|
|
|
|
void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
|
|
CHECK(!heap_start);
|
|
possible_regions.Init();
|
|
internal_memset(size_class_info_array, 0, sizeof(size_class_info_array));
|
|
}
|
|
|
|
s32 ReleaseToOSIntervalMs() const {
|
|
return kReleaseToOSIntervalNever;
|
|
}
|
|
|
|
void SetReleaseToOSIntervalMs(s32 release_to_os_interval_ms) {
|
|
// This is empty here. Currently only implemented in 64-bit allocator.
|
|
}
|
|
|
|
void ForceReleaseToOS() {
|
|
// Currently implemented in 64-bit allocator only.
|
|
}
|
|
|
|
void *MapWithCallback(uptr size) {
|
|
void *res = MmapOrDie(size, PrimaryAllocatorName);
|
|
MapUnmapCallback().OnMap((uptr)res, size);
|
|
return res;
|
|
}
|
|
|
|
void UnmapWithCallback(uptr beg, uptr size) {
|
|
MapUnmapCallback().OnUnmap(beg, size);
|
|
UnmapOrDie(reinterpret_cast<void *>(beg), size);
|
|
}
|
|
|
|
static bool CanAllocate(uptr size, uptr alignment) {
|
|
return size <= SizeClassMap::kMaxSize &&
|
|
alignment <= SizeClassMap::kMaxSize;
|
|
}
|
|
|
|
void *GetMetaData(const void *p) {
|
|
CHECK(kMetadataSize);
|
|
CHECK(PointerIsMine(p));
|
|
uptr mem = reinterpret_cast<uptr>(p);
|
|
uptr beg = ComputeRegionBeg(mem);
|
|
uptr size = ClassIdToSize(GetSizeClass(p));
|
|
u32 offset = mem - beg;
|
|
uptr n = offset / (u32)size; // 32-bit division
|
|
uptr meta = (beg + kRegionSize) - (n + 1) * kMetadataSize;
|
|
return reinterpret_cast<void*>(meta);
|
|
}
|
|
|
|
NOINLINE TransferBatch *AllocateBatch(AllocatorStats *stat, AllocatorCache *c,
|
|
uptr class_id) {
|
|
DCHECK_LT(class_id, kNumClasses);
|
|
SizeClassInfo *sci = GetSizeClassInfo(class_id);
|
|
SpinMutexLock l(&sci->mutex);
|
|
if (sci->free_list.empty()) {
|
|
if (UNLIKELY(!PopulateFreeList(stat, c, sci, class_id)))
|
|
return nullptr;
|
|
DCHECK(!sci->free_list.empty());
|
|
}
|
|
TransferBatch *b = sci->free_list.front();
|
|
sci->free_list.pop_front();
|
|
return b;
|
|
}
|
|
|
|
NOINLINE void DeallocateBatch(AllocatorStats *stat, uptr class_id,
|
|
TransferBatch *b) {
|
|
DCHECK_LT(class_id, kNumClasses);
|
|
CHECK_GT(b->Count(), 0);
|
|
SizeClassInfo *sci = GetSizeClassInfo(class_id);
|
|
SpinMutexLock l(&sci->mutex);
|
|
sci->free_list.push_front(b);
|
|
}
|
|
|
|
bool PointerIsMine(const void *p) {
|
|
uptr mem = reinterpret_cast<uptr>(p);
|
|
if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
|
|
mem &= (kSpaceSize - 1);
|
|
if (mem < kSpaceBeg || mem >= kSpaceBeg + kSpaceSize)
|
|
return false;
|
|
return GetSizeClass(p) != 0;
|
|
}
|
|
|
|
uptr GetSizeClass(const void *p) {
|
|
return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];
|
|
}
|
|
|
|
void *GetBlockBegin(const void *p) {
|
|
CHECK(PointerIsMine(p));
|
|
uptr mem = reinterpret_cast<uptr>(p);
|
|
uptr beg = ComputeRegionBeg(mem);
|
|
uptr size = ClassIdToSize(GetSizeClass(p));
|
|
u32 offset = mem - beg;
|
|
u32 n = offset / (u32)size; // 32-bit division
|
|
uptr res = beg + (n * (u32)size);
|
|
return reinterpret_cast<void*>(res);
|
|
}
|
|
|
|
uptr GetActuallyAllocatedSize(void *p) {
|
|
CHECK(PointerIsMine(p));
|
|
return ClassIdToSize(GetSizeClass(p));
|
|
}
|
|
|
|
static uptr ClassID(uptr size) { return SizeClassMap::ClassID(size); }
|
|
|
|
uptr TotalMemoryUsed() {
|
|
// No need to lock here.
|
|
uptr res = 0;
|
|
for (uptr i = 0; i < kNumPossibleRegions; i++)
|
|
if (possible_regions[i])
|
|
res += kRegionSize;
|
|
return res;
|
|
}
|
|
|
|
void TestOnlyUnmap() {
|
|
for (uptr i = 0; i < kNumPossibleRegions; i++)
|
|
if (possible_regions[i])
|
|
UnmapWithCallback((i * kRegionSize), kRegionSize);
|
|
}
|
|
|
|
// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
|
|
// introspection API.
|
|
void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
|
|
for (uptr i = 0; i < kNumClasses; i++) {
|
|
GetSizeClassInfo(i)->mutex.Lock();
|
|
}
|
|
}
|
|
|
|
void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
|
|
for (int i = kNumClasses - 1; i >= 0; i--) {
|
|
GetSizeClassInfo(i)->mutex.Unlock();
|
|
}
|
|
}
|
|
|
|
// Iterate over all existing chunks.
|
|
// The allocator must be locked when calling this function.
|
|
void ForEachChunk(ForEachChunkCallback callback, void *arg) {
|
|
for (uptr region = 0; region < kNumPossibleRegions; region++)
|
|
if (possible_regions[region]) {
|
|
uptr chunk_size = ClassIdToSize(possible_regions[region]);
|
|
uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
|
|
uptr region_beg = region * kRegionSize;
|
|
for (uptr chunk = region_beg;
|
|
chunk < region_beg + max_chunks_in_region * chunk_size;
|
|
chunk += chunk_size) {
|
|
// Too slow: CHECK_EQ((void *)chunk, GetBlockBegin((void *)chunk));
|
|
callback(chunk, arg);
|
|
}
|
|
}
|
|
}
|
|
|
|
void PrintStats() {}
|
|
|
|
static uptr AdditionalSize() { return 0; }
|
|
|
|
typedef SizeClassMap SizeClassMapT;
|
|
static const uptr kNumClasses = SizeClassMap::kNumClasses;
|
|
|
|
private:
|
|
static const uptr kRegionSize = 1 << kRegionSizeLog;
|
|
static const uptr kNumPossibleRegions = kSpaceSize / kRegionSize;
|
|
|
|
struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) SizeClassInfo {
|
|
StaticSpinMutex mutex;
|
|
IntrusiveList<TransferBatch> free_list;
|
|
u32 rand_state;
|
|
};
|
|
COMPILER_CHECK(sizeof(SizeClassInfo) % kCacheLineSize == 0);
|
|
|
|
uptr ComputeRegionId(uptr mem) const {
|
|
if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
|
|
mem &= (kSpaceSize - 1);
|
|
const uptr res = mem >> kRegionSizeLog;
|
|
CHECK_LT(res, kNumPossibleRegions);
|
|
return res;
|
|
}
|
|
|
|
uptr ComputeRegionBeg(uptr mem) {
|
|
return mem & ~(kRegionSize - 1);
|
|
}
|
|
|
|
uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
|
|
DCHECK_LT(class_id, kNumClasses);
|
|
const uptr res = reinterpret_cast<uptr>(MmapAlignedOrDieOnFatalError(
|
|
kRegionSize, kRegionSize, PrimaryAllocatorName));
|
|
if (UNLIKELY(!res))
|
|
return 0;
|
|
MapUnmapCallback().OnMap(res, kRegionSize);
|
|
stat->Add(AllocatorStatMapped, kRegionSize);
|
|
CHECK(IsAligned(res, kRegionSize));
|
|
possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
|
|
return res;
|
|
}
|
|
|
|
SizeClassInfo *GetSizeClassInfo(uptr class_id) {
|
|
DCHECK_LT(class_id, kNumClasses);
|
|
return &size_class_info_array[class_id];
|
|
}
|
|
|
|
bool PopulateBatches(AllocatorCache *c, SizeClassInfo *sci, uptr class_id,
|
|
TransferBatch **current_batch, uptr max_count,
|
|
uptr *pointers_array, uptr count) {
|
|
// If using a separate class for batches, we do not need to shuffle it.
|
|
if (kRandomShuffleChunks && (!kUseSeparateSizeClassForBatch ||
|
|
class_id != SizeClassMap::kBatchClassID))
|
|
RandomShuffle(pointers_array, count, &sci->rand_state);
|
|
TransferBatch *b = *current_batch;
|
|
for (uptr i = 0; i < count; i++) {
|
|
if (!b) {
|
|
b = c->CreateBatch(class_id, this, (TransferBatch*)pointers_array[i]);
|
|
if (UNLIKELY(!b))
|
|
return false;
|
|
b->Clear();
|
|
}
|
|
b->Add((void*)pointers_array[i]);
|
|
if (b->Count() == max_count) {
|
|
sci->free_list.push_back(b);
|
|
b = nullptr;
|
|
}
|
|
}
|
|
*current_batch = b;
|
|
return true;
|
|
}
|
|
|
|
bool PopulateFreeList(AllocatorStats *stat, AllocatorCache *c,
|
|
SizeClassInfo *sci, uptr class_id) {
|
|
const uptr region = AllocateRegion(stat, class_id);
|
|
if (UNLIKELY(!region))
|
|
return false;
|
|
if (kRandomShuffleChunks)
|
|
if (UNLIKELY(sci->rand_state == 0))
|
|
// The random state is initialized from ASLR (PIE) and time.
|
|
sci->rand_state = reinterpret_cast<uptr>(sci) ^ NanoTime();
|
|
const uptr size = ClassIdToSize(class_id);
|
|
const uptr n_chunks = kRegionSize / (size + kMetadataSize);
|
|
const uptr max_count = TransferBatch::MaxCached(size);
|
|
DCHECK_GT(max_count, 0);
|
|
TransferBatch *b = nullptr;
|
|
constexpr uptr kShuffleArraySize = 48;
|
|
uptr shuffle_array[kShuffleArraySize];
|
|
uptr count = 0;
|
|
for (uptr i = region; i < region + n_chunks * size; i += size) {
|
|
shuffle_array[count++] = i;
|
|
if (count == kShuffleArraySize) {
|
|
if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
|
|
shuffle_array, count)))
|
|
return false;
|
|
count = 0;
|
|
}
|
|
}
|
|
if (count) {
|
|
if (UNLIKELY(!PopulateBatches(c, sci, class_id, &b, max_count,
|
|
shuffle_array, count)))
|
|
return false;
|
|
}
|
|
if (b) {
|
|
CHECK_GT(b->Count(), 0);
|
|
sci->free_list.push_back(b);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
ByteMap possible_regions;
|
|
SizeClassInfo size_class_info_array[kNumClasses];
|
|
};
|