gcc/libphobos/libdruntime/gc/impl/conservative/gc.d
Iain Buclaw 151a199f29 libphobos: Add Fiber/Thread support for StackGrowsUp.
The StackGrowsDown version being turned off for hppa targets.

After other fixes in the compiler, this allows core.thread unittests to
all pass, as well as the garbage collector to work correctly.

Backported from upstream druntime 2.084.

Reviewed-on: https://github.com/dlang/druntime/pull/2410

From-SVN: r268056
2019-01-18 01:51:36 +00:00

3414 lines
92 KiB
D

/**
* Contains the garbage collector implementation.
*
* Copyright: Copyright Digital Mars 2001 - 2016.
* License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
* Authors: Walter Bright, David Friedman, Sean Kelly
*/
/* Copyright Digital Mars 2005 - 2016.
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE or copy at
* http://www.boost.org/LICENSE_1_0.txt)
*/
module gc.impl.conservative.gc;
// D Programming Language Garbage Collector implementation
/************** Debugging ***************************/
//debug = PRINTF; // turn on printf's
//debug = COLLECT_PRINTF; // turn on printf's
//debug = PRINTF_TO_FILE; // redirect printf's ouptut to file "gcx.log"
//debug = LOGGING; // log allocations / frees
//debug = MEMSTOMP; // stomp on memory
//debug = SENTINEL; // add underrun/overrrun protection
// NOTE: this needs to be enabled globally in the makefiles
// (-debug=SENTINEL) to pass druntime's unittests.
//debug = PTRCHECK; // more pointer checking
//debug = PTRCHECK2; // thorough but slow pointer checking
//debug = INVARIANT; // enable invariants
//debug = PROFILE_API; // profile API calls for config.profile > 1
/***************************************************/
import gc.bits;
import gc.os;
import gc.config;
import gc.gcinterface;
import rt.util.container.treap;
import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc;
import core.stdc.string : memcpy, memset, memmove;
import core.bitop;
import core.thread;
static import core.memory;
version (GNU) import gcc.builtins;
debug (PRINTF_TO_FILE) import core.stdc.stdio : sprintf, fprintf, fopen, fflush, FILE;
else import core.stdc.stdio : sprintf, printf; // needed to output profiling results
import core.time;
alias currTime = MonoTime.currTime;
debug(PRINTF_TO_FILE)
{
private __gshared MonoTime gcStartTick;
private __gshared FILE* gcx_fh;
private int printf(ARGS...)(const char* fmt, ARGS args) nothrow
{
if (!gcx_fh)
gcx_fh = fopen("gcx.log", "w");
if (!gcx_fh)
return 0;
int len;
if (MonoTime.ticksPerSecond == 0)
{
len = fprintf(gcx_fh, "before init: ");
}
else
{
if (gcStartTick == MonoTime.init)
gcStartTick = MonoTime.currTime;
immutable timeElapsed = MonoTime.currTime - gcStartTick;
immutable secondsAsDouble = timeElapsed.total!"hnsecs" / cast(double)convert!("seconds", "hnsecs")(1);
len = fprintf(gcx_fh, "%10.6lf: ", secondsAsDouble);
}
len += fprintf(gcx_fh, fmt, args);
fflush(gcx_fh);
return len;
}
}
debug(PRINTF) void printFreeInfo(Pool* pool) nothrow
{
uint nReallyFree;
foreach (i; 0..pool.npages) {
if (pool.pagetable[i] >= B_FREE) nReallyFree++;
}
printf("Pool %p: %d really free, %d supposedly free\n", pool, nReallyFree, pool.freepages);
}
// Track total time spent preparing for GC,
// marking, sweeping and recovering pages.
__gshared Duration prepTime;
__gshared Duration markTime;
__gshared Duration sweepTime;
__gshared Duration recoverTime;
__gshared Duration maxPauseTime;
__gshared size_t numCollections;
__gshared size_t maxPoolMemory;
__gshared long numMallocs;
__gshared long numFrees;
__gshared long numReallocs;
__gshared long numExtends;
__gshared long numOthers;
__gshared long mallocTime; // using ticks instead of MonoTime for better performance
__gshared long freeTime;
__gshared long reallocTime;
__gshared long extendTime;
__gshared long otherTime;
__gshared long lockTime;
private
{
extern (C)
{
// to allow compilation of this module without access to the rt package,
// make these functions available from rt.lifetime
void rt_finalizeFromGC(void* p, size_t size, uint attr) nothrow;
int rt_hasFinalizerInSegment(void* p, size_t size, uint attr, in void[] segment) nothrow;
// Declared as an extern instead of importing core.exception
// to avoid inlining - see issue 13725.
void onInvalidMemoryOperationError() @nogc nothrow;
void onOutOfMemoryErrorNoGC() @nogc nothrow;
}
enum
{
OPFAIL = ~cast(size_t)0
}
}
alias GC gc_t;
/* ======================= Leak Detector =========================== */
debug (LOGGING)
{
struct Log
{
void* p;
size_t size;
size_t line;
char* file;
void* parent;
void print() nothrow
{
printf(" p = %p, size = %zd, parent = %p ", p, size, parent);
if (file)
{
printf("%s(%u)", file, line);
}
printf("\n");
}
}
struct LogArray
{
size_t dim;
size_t allocdim;
Log *data;
void Dtor() nothrow
{
if (data)
cstdlib.free(data);
data = null;
}
void reserve(size_t nentries) nothrow
{
assert(dim <= allocdim);
if (allocdim - dim < nentries)
{
allocdim = (dim + nentries) * 2;
assert(dim + nentries <= allocdim);
if (!data)
{
data = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof);
if (!data && allocdim)
onOutOfMemoryErrorNoGC();
}
else
{ Log *newdata;
newdata = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof);
if (!newdata && allocdim)
onOutOfMemoryErrorNoGC();
memcpy(newdata, data, dim * Log.sizeof);
cstdlib.free(data);
data = newdata;
}
}
}
void push(Log log) nothrow
{
reserve(1);
data[dim++] = log;
}
void remove(size_t i) nothrow
{
memmove(data + i, data + i + 1, (dim - i) * Log.sizeof);
dim--;
}
size_t find(void *p) nothrow
{
for (size_t i = 0; i < dim; i++)
{
if (data[i].p == p)
return i;
}
return OPFAIL; // not found
}
void copy(LogArray *from) nothrow
{
reserve(from.dim - dim);
assert(from.dim <= allocdim);
memcpy(data, from.data, from.dim * Log.sizeof);
dim = from.dim;
}
}
}
/* ============================ GC =============================== */
class ConservativeGC : GC
{
// For passing to debug code (not thread safe)
__gshared size_t line;
__gshared char* file;
Gcx *gcx; // implementation
import core.internal.spinlock;
static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy);
static bool _inFinalizer;
// lock GC, throw InvalidMemoryOperationError on recursive locking during finalization
static void lockNR() @nogc nothrow
{
if (_inFinalizer)
onInvalidMemoryOperationError();
gcLock.lock();
}
static void initialize(ref GC gc)
{
import core.stdc.string: memcpy;
if (config.gc != "conservative")
return;
auto p = cstdlib.malloc(__traits(classInstanceSize,ConservativeGC));
if (!p)
onOutOfMemoryErrorNoGC();
auto init = typeid(ConservativeGC).initializer();
assert(init.length == __traits(classInstanceSize, ConservativeGC));
auto instance = cast(ConservativeGC) memcpy(p, init.ptr, init.length);
instance.__ctor();
gc = instance;
}
static void finalize(ref GC gc)
{
if (config.gc != "conservative")
return;
auto instance = cast(ConservativeGC) gc;
instance.Dtor();
cstdlib.free(cast(void*)instance);
}
this()
{
//config is assumed to have already been initialized
gcx = cast(Gcx*)cstdlib.calloc(1, Gcx.sizeof);
if (!gcx)
onOutOfMemoryErrorNoGC();
gcx.initialize();
if (config.initReserve)
gcx.reserve(config.initReserve << 20);
if (config.disable)
gcx.disabled++;
}
void Dtor()
{
version (linux)
{
//debug(PRINTF) printf("Thread %x ", pthread_self());
//debug(PRINTF) printf("GC.Dtor()\n");
}
if (gcx)
{
gcx.Dtor();
cstdlib.free(gcx);
gcx = null;
}
}
void enable()
{
static void go(Gcx* gcx) nothrow
{
assert(gcx.disabled > 0);
gcx.disabled--;
}
runLocked!(go, otherTime, numOthers)(gcx);
}
void disable()
{
static void go(Gcx* gcx) nothrow
{
gcx.disabled++;
}
runLocked!(go, otherTime, numOthers)(gcx);
}
auto runLocked(alias func, Args...)(auto ref Args args)
{
debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0);
lockNR();
scope (failure) gcLock.unlock();
debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0);
static if (is(typeof(func(args)) == void))
func(args);
else
auto res = func(args);
debug(PROFILE_API) if (config.profile > 1)
lockTime += tm2 - tm;
gcLock.unlock();
static if (!is(typeof(func(args)) == void))
return res;
}
auto runLocked(alias func, alias time, alias count, Args...)(auto ref Args args)
{
debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0);
lockNR();
scope (failure) gcLock.unlock();
debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0);
static if (is(typeof(func(args)) == void))
func(args);
else
auto res = func(args);
debug(PROFILE_API) if (config.profile > 1)
{
count++;
immutable now = currTime.ticks;
lockTime += tm2 - tm;
time += now - tm2;
}
gcLock.unlock();
static if (!is(typeof(func(args)) == void))
return res;
}
uint getAttr(void* p) nothrow
{
if (!p)
{
return 0;
}
static uint go(Gcx* gcx, void* p) nothrow
{
Pool* pool = gcx.findPool(p);
uint oldb = 0;
if (pool)
{
p = sentinel_sub(p);
auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
oldb = pool.getBits(biti);
}
return oldb;
}
return runLocked!(go, otherTime, numOthers)(gcx, p);
}
uint setAttr(void* p, uint mask) nothrow
{
if (!p)
{
return 0;
}
static uint go(Gcx* gcx, void* p, uint mask) nothrow
{
Pool* pool = gcx.findPool(p);
uint oldb = 0;
if (pool)
{
p = sentinel_sub(p);
auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
oldb = pool.getBits(biti);
pool.setBits(biti, mask);
}
return oldb;
}
return runLocked!(go, otherTime, numOthers)(gcx, p, mask);
}
uint clrAttr(void* p, uint mask) nothrow
{
if (!p)
{
return 0;
}
static uint go(Gcx* gcx, void* p, uint mask) nothrow
{
Pool* pool = gcx.findPool(p);
uint oldb = 0;
if (pool)
{
p = sentinel_sub(p);
auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
oldb = pool.getBits(biti);
pool.clrBits(biti, mask);
}
return oldb;
}
return runLocked!(go, otherTime, numOthers)(gcx, p, mask);
}
void *malloc(size_t size, uint bits, const TypeInfo ti) nothrow
{
if (!size)
{
return null;
}
size_t localAllocSize = void;
auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti);
if (!(bits & BlkAttr.NO_SCAN))
{
memset(p + size, 0, localAllocSize - size);
}
return p;
}
//
//
//
private void *mallocNoSync(size_t size, uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow
{
assert(size != 0);
//debug(PRINTF) printf("GC::malloc(size = %d, gcx = %p)\n", size, gcx);
assert(gcx);
//debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self());
auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits);
if (!p)
onOutOfMemoryErrorNoGC();
debug (SENTINEL)
{
p = sentinel_add(p);
sentinel_init(p, size);
alloc_size = size;
}
gcx.log_malloc(p, size);
return p;
}
BlkInfo qalloc( size_t size, uint bits, const TypeInfo ti) nothrow
{
if (!size)
{
return BlkInfo.init;
}
BlkInfo retval;
retval.base = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, retval.size, ti);
if (!(bits & BlkAttr.NO_SCAN))
{
memset(retval.base + size, 0, retval.size - size);
}
retval.attr = bits;
return retval;
}
void *calloc(size_t size, uint bits, const TypeInfo ti) nothrow
{
if (!size)
{
return null;
}
size_t localAllocSize = void;
auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti);
memset(p, 0, size);
if (!(bits & BlkAttr.NO_SCAN))
{
memset(p + size, 0, localAllocSize - size);
}
return p;
}
void *realloc(void *p, size_t size, uint bits, const TypeInfo ti) nothrow
{
size_t localAllocSize = void;
auto oldp = p;
p = runLocked!(reallocNoSync, mallocTime, numMallocs)(p, size, bits, localAllocSize, ti);
if (p !is oldp && !(bits & BlkAttr.NO_SCAN))
{
memset(p + size, 0, localAllocSize - size);
}
return p;
}
//
// bits will be set to the resulting bits of the new block
//
private void *reallocNoSync(void *p, size_t size, ref uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow
{
if (!size)
{ if (p)
{ freeNoSync(p);
p = null;
}
alloc_size = 0;
}
else if (!p)
{
p = mallocNoSync(size, bits, alloc_size, ti);
}
else
{ void *p2;
size_t psize;
//debug(PRINTF) printf("GC::realloc(p = %p, size = %zu)\n", p, size);
debug (SENTINEL)
{
sentinel_Invariant(p);
psize = *sentinel_size(p);
if (psize != size)
{
if (psize)
{
Pool *pool = gcx.findPool(p);
if (pool)
{
auto biti = cast(size_t)(sentinel_sub(p) - pool.baseAddr) >> pool.shiftBy;
if (bits)
{
pool.clrBits(biti, ~BlkAttr.NONE);
pool.setBits(biti, bits);
}
else
{
bits = pool.getBits(biti);
}
}
}
p2 = mallocNoSync(size, bits, alloc_size, ti);
if (psize < size)
size = psize;
//debug(PRINTF) printf("\tcopying %d bytes\n",size);
memcpy(p2, p, size);
p = p2;
}
}
else
{
auto pool = gcx.findPool(p);
if (pool.isLargeObject)
{
auto lpool = cast(LargeObjectPool*) pool;
psize = lpool.getSize(p); // get allocated size
if (size <= PAGESIZE / 2)
goto Lmalloc; // switching from large object pool to small object pool
auto psz = psize / PAGESIZE;
auto newsz = (size + PAGESIZE - 1) / PAGESIZE;
if (newsz == psz)
{
alloc_size = psize;
return p;
}
auto pagenum = lpool.pagenumOf(p);
if (newsz < psz)
{ // Shrink in place
debug (MEMSTOMP) memset(p + size, 0xF2, psize - size);
lpool.freePages(pagenum + newsz, psz - newsz);
}
else if (pagenum + newsz <= pool.npages)
{ // Attempt to expand in place
foreach (binsz; lpool.pagetable[pagenum + psz .. pagenum + newsz])
if (binsz != B_FREE)
goto Lmalloc;
debug (MEMSTOMP) memset(p + psize, 0xF0, size - psize);
debug(PRINTF) printFreeInfo(pool);
memset(&lpool.pagetable[pagenum + psz], B_PAGEPLUS, newsz - psz);
gcx.usedLargePages += newsz - psz;
lpool.freepages -= (newsz - psz);
debug(PRINTF) printFreeInfo(pool);
}
else
goto Lmalloc; // does not fit into current pool
lpool.updateOffsets(pagenum);
if (bits)
{
immutable biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
pool.clrBits(biti, ~BlkAttr.NONE);
pool.setBits(biti, bits);
}
alloc_size = newsz * PAGESIZE;
return p;
}
psize = (cast(SmallObjectPool*) pool).getSize(p); // get allocated size
if (psize < size || // if new size is bigger
psize > size * 2) // or less than half
{
Lmalloc:
if (psize && pool)
{
auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
if (bits)
{
pool.clrBits(biti, ~BlkAttr.NONE);
pool.setBits(biti, bits);
}
else
{
bits = pool.getBits(biti);
}
}
p2 = mallocNoSync(size, bits, alloc_size, ti);
if (psize < size)
size = psize;
//debug(PRINTF) printf("\tcopying %d bytes\n",size);
memcpy(p2, p, size);
p = p2;
}
else
alloc_size = psize;
}
}
return p;
}
size_t extend(void* p, size_t minsize, size_t maxsize, const TypeInfo ti) nothrow
{
return runLocked!(extendNoSync, extendTime, numExtends)(p, minsize, maxsize, ti);
}
//
//
//
private size_t extendNoSync(void* p, size_t minsize, size_t maxsize, const TypeInfo ti = null) nothrow
in
{
assert(minsize <= maxsize);
}
body
{
//debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize);
debug (SENTINEL)
{
return 0;
}
else
{
auto pool = gcx.findPool(p);
if (!pool || !pool.isLargeObject)
return 0;
auto lpool = cast(LargeObjectPool*) pool;
auto psize = lpool.getSize(p); // get allocated size
if (psize < PAGESIZE)
return 0; // cannot extend buckets
auto psz = psize / PAGESIZE;
auto minsz = (minsize + PAGESIZE - 1) / PAGESIZE;
auto maxsz = (maxsize + PAGESIZE - 1) / PAGESIZE;
auto pagenum = lpool.pagenumOf(p);
size_t sz;
for (sz = 0; sz < maxsz; sz++)
{
auto i = pagenum + psz + sz;
if (i == lpool.npages)
break;
if (lpool.pagetable[i] != B_FREE)
{ if (sz < minsz)
return 0;
break;
}
}
if (sz < minsz)
return 0;
debug (MEMSTOMP) memset(pool.baseAddr + (pagenum + psz) * PAGESIZE, 0xF0, sz * PAGESIZE);
memset(lpool.pagetable + pagenum + psz, B_PAGEPLUS, sz);
lpool.updateOffsets(pagenum);
lpool.freepages -= sz;
gcx.usedLargePages += sz;
return (psz + sz) * PAGESIZE;
}
}
size_t reserve(size_t size) nothrow
{
if (!size)
{
return 0;
}
return runLocked!(reserveNoSync, otherTime, numOthers)(size);
}
//
//
//
private size_t reserveNoSync(size_t size) nothrow
{
assert(size != 0);
assert(gcx);
return gcx.reserve(size);
}
void free(void *p) nothrow
{
if (!p || _inFinalizer)
{
return;
}
return runLocked!(freeNoSync, freeTime, numFrees)(p);
}
//
//
//
private void freeNoSync(void *p) nothrow
{
debug(PRINTF) printf("Freeing %p\n", cast(size_t) p);
assert (p);
Pool* pool;
size_t pagenum;
Bins bin;
size_t biti;
// Find which page it is in
pool = gcx.findPool(p);
if (!pool) // if not one of ours
return; // ignore
pagenum = pool.pagenumOf(p);
debug(PRINTF) printf("pool base = %p, PAGENUM = %d of %d, bin = %d\n", pool.baseAddr, pagenum, pool.npages, pool.pagetable[pagenum]);
debug(PRINTF) if (pool.isLargeObject) printf("Block size = %d\n", pool.bPageOffsets[pagenum]);
bin = cast(Bins)pool.pagetable[pagenum];
// Verify that the pointer is at the beginning of a block,
// no action should be taken if p is an interior pointer
if (bin > B_PAGE) // B_PAGEPLUS or B_FREE
return;
if ((sentinel_sub(p) - pool.baseAddr) & (binsize[bin] - 1))
return;
sentinel_Invariant(p);
p = sentinel_sub(p);
biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy;
pool.clrBits(biti, ~BlkAttr.NONE);
if (pool.isLargeObject) // if large alloc
{
assert(bin == B_PAGE);
auto lpool = cast(LargeObjectPool*) pool;
// Free pages
size_t npages = lpool.bPageOffsets[pagenum];
debug (MEMSTOMP) memset(p, 0xF2, npages * PAGESIZE);
lpool.freePages(pagenum, npages);
}
else
{ // Add to free list
List *list = cast(List*)p;
debug (MEMSTOMP) memset(p, 0xF2, binsize[bin]);
list.next = gcx.bucket[bin];
list.pool = pool;
gcx.bucket[bin] = list;
}
gcx.log_free(sentinel_add(p));
}
void* addrOf(void *p) nothrow
{
if (!p)
{
return null;
}
return runLocked!(addrOfNoSync, otherTime, numOthers)(p);
}
//
//
//
void* addrOfNoSync(void *p) nothrow
{
if (!p)
{
return null;
}
auto q = gcx.findBase(p);
if (q)
q = sentinel_add(q);
return q;
}
size_t sizeOf(void *p) nothrow
{
if (!p)
{
return 0;
}
return runLocked!(sizeOfNoSync, otherTime, numOthers)(p);
}
//
//
//
private size_t sizeOfNoSync(void *p) nothrow
{
assert (p);
debug (SENTINEL)
{
p = sentinel_sub(p);
size_t size = gcx.findSize(p);
// Check for interior pointer
// This depends on:
// 1) size is a power of 2 for less than PAGESIZE values
// 2) base of memory pool is aligned on PAGESIZE boundary
if (cast(size_t)p & (size - 1) & (PAGESIZE - 1))
size = 0;
return size ? size - SENTINEL_EXTRA : 0;
}
else
{
size_t size = gcx.findSize(p);
// Check for interior pointer
// This depends on:
// 1) size is a power of 2 for less than PAGESIZE values
// 2) base of memory pool is aligned on PAGESIZE boundary
if (cast(size_t)p & (size - 1) & (PAGESIZE - 1))
return 0;
return size;
}
}
BlkInfo query(void *p) nothrow
{
if (!p)
{
BlkInfo i;
return i;
}
return runLocked!(queryNoSync, otherTime, numOthers)(p);
}
//
//
//
BlkInfo queryNoSync(void *p) nothrow
{
assert(p);
BlkInfo info = gcx.getInfo(p);
debug(SENTINEL)
{
if (info.base)
{
info.base = sentinel_add(info.base);
info.size = *sentinel_size(info.base);
}
}
return info;
}
/**
* Verify that pointer p:
* 1) belongs to this memory pool
* 2) points to the start of an allocated piece of memory
* 3) is not on a free list
*/
void check(void *p) nothrow
{
if (!p)
{
return;
}
return runLocked!(checkNoSync, otherTime, numOthers)(p);
}
//
//
//
private void checkNoSync(void *p) nothrow
{
assert(p);
sentinel_Invariant(p);
debug (PTRCHECK)
{
Pool* pool;
size_t pagenum;
Bins bin;
size_t size;
p = sentinel_sub(p);
pool = gcx.findPool(p);
assert(pool);
pagenum = pool.pagenumOf(p);
bin = cast(Bins)pool.pagetable[pagenum];
assert(bin <= B_PAGE);
size = binsize[bin];
assert((cast(size_t)p & (size - 1)) == 0);
debug (PTRCHECK2)
{
if (bin < B_PAGE)
{
// Check that p is not on a free list
List *list;
for (list = gcx.bucket[bin]; list; list = list.next)
{
assert(cast(void*)list != p);
}
}
}
}
}
void addRoot(void *p) nothrow @nogc
{
if (!p)
{
return;
}
gcx.addRoot(p);
}
void removeRoot(void *p) nothrow @nogc
{
if (!p)
{
return;
}
gcx.removeRoot(p);
}
@property RootIterator rootIter() @nogc
{
return &gcx.rootsApply;
}
void addRange(void *p, size_t sz, const TypeInfo ti = null) nothrow @nogc
{
if (!p || !sz)
{
return;
}
gcx.addRange(p, p + sz, ti);
}
void removeRange(void *p) nothrow @nogc
{
if (!p)
{
return;
}
gcx.removeRange(p);
}
@property RangeIterator rangeIter() @nogc
{
return &gcx.rangesApply;
}
void runFinalizers(in void[] segment) nothrow
{
static void go(Gcx* gcx, in void[] segment) nothrow
{
gcx.runFinalizers(segment);
}
return runLocked!(go, otherTime, numOthers)(gcx, segment);
}
bool inFinalizer() nothrow
{
return _inFinalizer;
}
void collect() nothrow
{
fullCollect();
}
void collectNoStack() nothrow
{
fullCollectNoStack();
}
/**
* Do full garbage collection.
* Return number of pages free'd.
*/
size_t fullCollect() nothrow
{
debug(PRINTF) printf("GC.fullCollect()\n");
// Since a finalizer could launch a new thread, we always need to lock
// when collecting.
static size_t go(Gcx* gcx) nothrow
{
return gcx.fullcollect();
}
immutable result = runLocked!go(gcx);
version (none)
{
GCStats stats;
getStats(stats);
debug(PRINTF) printf("heapSize = %zx, freeSize = %zx\n",
stats.heapSize, stats.freeSize);
}
gcx.log_collect();
return result;
}
/**
* do full garbage collection ignoring roots
*/
void fullCollectNoStack() nothrow
{
// Since a finalizer could launch a new thread, we always need to lock
// when collecting.
static size_t go(Gcx* gcx) nothrow
{
return gcx.fullcollect(true);
}
runLocked!go(gcx);
}
void minimize() nothrow
{
static void go(Gcx* gcx) nothrow
{
gcx.minimize();
}
runLocked!(go, otherTime, numOthers)(gcx);
}
core.memory.GC.Stats stats() nothrow
{
typeof(return) ret;
runLocked!(getStatsNoSync, otherTime, numOthers)(ret);
return ret;
}
//
//
//
private void getStatsNoSync(out core.memory.GC.Stats stats) nothrow
{
foreach (pool; gcx.pooltable[0 .. gcx.npools])
{
foreach (bin; pool.pagetable[0 .. pool.npages])
{
if (bin == B_FREE)
stats.freeSize += PAGESIZE;
else
stats.usedSize += PAGESIZE;
}
}
size_t freeListSize;
foreach (n; 0 .. B_PAGE)
{
immutable sz = binsize[n];
for (List *list = gcx.bucket[n]; list; list = list.next)
freeListSize += sz;
}
stats.usedSize -= freeListSize;
stats.freeSize += freeListSize;
}
}
/* ============================ Gcx =============================== */
enum
{ PAGESIZE = 4096,
POOLSIZE = (4096*256),
}
enum
{
B_16,
B_32,
B_64,
B_128,
B_256,
B_512,
B_1024,
B_2048,
B_PAGE, // start of large alloc
B_PAGEPLUS, // continuation of large alloc
B_FREE, // free page
B_MAX
}
alias ubyte Bins;
struct List
{
List *next;
Pool *pool;
}
immutable uint[B_MAX] binsize = [ 16,32,64,128,256,512,1024,2048,4096 ];
immutable size_t[B_MAX] notbinsize = [ ~(16-1),~(32-1),~(64-1),~(128-1),~(256-1),
~(512-1),~(1024-1),~(2048-1),~(4096-1) ];
alias PageBits = GCBits.wordtype[PAGESIZE / 16 / GCBits.BITS_PER_WORD];
static assert(PAGESIZE % (GCBits.BITS_PER_WORD * 16) == 0);
private void set(ref PageBits bits, size_t i) @nogc pure nothrow
{
assert(i < PageBits.sizeof * 8);
bts(bits.ptr, i);
}
/* ============================ Gcx =============================== */
struct Gcx
{
import core.internal.spinlock;
auto rootsLock = shared(AlignedSpinLock)(SpinLock.Contention.brief);
auto rangesLock = shared(AlignedSpinLock)(SpinLock.Contention.brief);
Treap!Root roots;
Treap!Range ranges;
bool log; // turn on logging
debug(INVARIANT) bool initialized;
uint disabled; // turn off collections if >0
import gc.pooltable;
@property size_t npools() pure const nothrow { return pooltable.length; }
PoolTable!Pool pooltable;
List*[B_PAGE] bucket; // free list for each small size
// run a collection when reaching those thresholds (number of used pages)
float smallCollectThreshold, largeCollectThreshold;
uint usedSmallPages, usedLargePages;
// total number of mapped pages
uint mappedPages;
void initialize()
{
(cast(byte*)&this)[0 .. Gcx.sizeof] = 0;
log_init();
roots.initialize();
ranges.initialize();
smallCollectThreshold = largeCollectThreshold = 0.0f;
usedSmallPages = usedLargePages = 0;
mappedPages = 0;
//printf("gcx = %p, self = %x\n", &this, self);
debug(INVARIANT) initialized = true;
}
void Dtor()
{
if (config.profile)
{
printf("\tNumber of collections: %llu\n", cast(ulong)numCollections);
printf("\tTotal GC prep time: %lld milliseconds\n",
prepTime.total!("msecs"));
printf("\tTotal mark time: %lld milliseconds\n",
markTime.total!("msecs"));
printf("\tTotal sweep time: %lld milliseconds\n",
sweepTime.total!("msecs"));
printf("\tTotal page recovery time: %lld milliseconds\n",
recoverTime.total!("msecs"));
long maxPause = maxPauseTime.total!("msecs");
printf("\tMax Pause Time: %lld milliseconds\n", maxPause);
long gcTime = (recoverTime + sweepTime + markTime + prepTime).total!("msecs");
printf("\tGrand total GC time: %lld milliseconds\n", gcTime);
long pauseTime = (markTime + prepTime).total!("msecs");
char[30] apitxt;
apitxt[0] = 0;
debug(PROFILE_API) if (config.profile > 1)
{
static Duration toDuration(long dur)
{
return MonoTime(dur) - MonoTime(0);
}
printf("\n");
printf("\tmalloc: %llu calls, %lld ms\n", cast(ulong)numMallocs, toDuration(mallocTime).total!"msecs");
printf("\trealloc: %llu calls, %lld ms\n", cast(ulong)numReallocs, toDuration(reallocTime).total!"msecs");
printf("\tfree: %llu calls, %lld ms\n", cast(ulong)numFrees, toDuration(freeTime).total!"msecs");
printf("\textend: %llu calls, %lld ms\n", cast(ulong)numExtends, toDuration(extendTime).total!"msecs");
printf("\tother: %llu calls, %lld ms\n", cast(ulong)numOthers, toDuration(otherTime).total!"msecs");
printf("\tlock time: %lld ms\n", toDuration(lockTime).total!"msecs");
long apiTime = mallocTime + reallocTime + freeTime + extendTime + otherTime + lockTime;
printf("\tGC API: %lld ms\n", toDuration(apiTime).total!"msecs");
sprintf(apitxt.ptr, " API%5ld ms", toDuration(apiTime).total!"msecs");
}
printf("GC summary:%5lld MB,%5lld GC%5lld ms, Pauses%5lld ms <%5lld ms%s\n",
cast(long) maxPoolMemory >> 20, cast(ulong)numCollections, gcTime,
pauseTime, maxPause, apitxt.ptr);
}
debug(INVARIANT) initialized = false;
for (size_t i = 0; i < npools; i++)
{
Pool *pool = pooltable[i];
mappedPages -= pool.npages;
pool.Dtor();
cstdlib.free(pool);
}
assert(!mappedPages);
pooltable.Dtor();
roots.removeAll();
ranges.removeAll();
toscan.reset();
}
void Invariant() const { }
debug(INVARIANT)
invariant()
{
if (initialized)
{
//printf("Gcx.invariant(): this = %p\n", &this);
pooltable.Invariant();
rangesLock.lock();
foreach (range; ranges)
{
assert(range.pbot);
assert(range.ptop);
assert(range.pbot <= range.ptop);
}
rangesLock.unlock();
for (size_t i = 0; i < B_PAGE; i++)
{
for (auto list = cast(List*)bucket[i]; list; list = list.next)
{
}
}
}
}
/**
*
*/
void addRoot(void *p) nothrow @nogc
{
rootsLock.lock();
scope (failure) rootsLock.unlock();
roots.insert(Root(p));
rootsLock.unlock();
}
/**
*
*/
void removeRoot(void *p) nothrow @nogc
{
rootsLock.lock();
scope (failure) rootsLock.unlock();
roots.remove(Root(p));
rootsLock.unlock();
}
/**
*
*/
int rootsApply(scope int delegate(ref Root) nothrow dg) nothrow
{
rootsLock.lock();
scope (failure) rootsLock.unlock();
auto ret = roots.opApply(dg);
rootsLock.unlock();
return ret;
}
/**
*
*/
void addRange(void *pbot, void *ptop, const TypeInfo ti) nothrow @nogc
{
//debug(PRINTF) printf("Thread %x ", pthread_self());
debug(PRINTF) printf("%p.Gcx::addRange(%p, %p)\n", &this, pbot, ptop);
rangesLock.lock();
scope (failure) rangesLock.unlock();
ranges.insert(Range(pbot, ptop));
rangesLock.unlock();
}
/**
*
*/
void removeRange(void *pbot) nothrow @nogc
{
//debug(PRINTF) printf("Thread %x ", pthread_self());
debug(PRINTF) printf("Gcx.removeRange(%p)\n", pbot);
rangesLock.lock();
scope (failure) rangesLock.unlock();
ranges.remove(Range(pbot, pbot)); // only pbot is used, see Range.opCmp
rangesLock.unlock();
// debug(PRINTF) printf("Wrong thread\n");
// This is a fatal error, but ignore it.
// The problem is that we can get a Close() call on a thread
// other than the one the range was allocated on.
//assert(zero);
}
/**
*
*/
int rangesApply(scope int delegate(ref Range) nothrow dg) nothrow
{
rangesLock.lock();
scope (failure) rangesLock.unlock();
auto ret = ranges.opApply(dg);
rangesLock.unlock();
return ret;
}
/**
*
*/
void runFinalizers(in void[] segment) nothrow
{
ConservativeGC._inFinalizer = true;
scope (failure) ConservativeGC._inFinalizer = false;
foreach (pool; pooltable[0 .. npools])
{
if (!pool.finals.nbits) continue;
if (pool.isLargeObject)
{
auto lpool = cast(LargeObjectPool*) pool;
lpool.runFinalizers(segment);
}
else
{
auto spool = cast(SmallObjectPool*) pool;
spool.runFinalizers(segment);
}
}
ConservativeGC._inFinalizer = false;
}
Pool* findPool(void* p) pure nothrow
{
return pooltable.findPool(p);
}
/**
* Find base address of block containing pointer p.
* Returns null if not a gc'd pointer
*/
void* findBase(void *p) nothrow
{
Pool *pool;
pool = findPool(p);
if (pool)
{
size_t offset = cast(size_t)(p - pool.baseAddr);
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pool.pagetable[pn];
// Adjust bit to be at start of allocated memory block
if (bin <= B_PAGE)
{
return pool.baseAddr + (offset & notbinsize[bin]);
}
else if (bin == B_PAGEPLUS)
{
auto pageOffset = pool.bPageOffsets[pn];
offset -= pageOffset * PAGESIZE;
pn -= pageOffset;
return pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1)));
}
else
{
// we are in a B_FREE page
assert(bin == B_FREE);
return null;
}
}
return null;
}
/**
* Find size of pointer p.
* Returns 0 if not a gc'd pointer
*/
size_t findSize(void *p) nothrow
{
Pool* pool = findPool(p);
if (pool)
return pool.slGetSize(p);
return 0;
}
/**
*
*/
BlkInfo getInfo(void* p) nothrow
{
Pool* pool = findPool(p);
if (pool)
return pool.slGetInfo(p);
return BlkInfo();
}
/**
* Computes the bin table using CTFE.
*/
static byte[2049] ctfeBins() nothrow
{
byte[2049] ret;
size_t p = 0;
for (Bins b = B_16; b <= B_2048; b++)
for ( ; p <= binsize[b]; p++)
ret[p] = b;
return ret;
}
static const byte[2049] binTable = ctfeBins();
/**
* Allocate a new pool of at least size bytes.
* Sort it into pooltable[].
* Mark all memory in the pool as B_FREE.
* Return the actual number of bytes reserved or 0 on error.
*/
size_t reserve(size_t size) nothrow
{
size_t npages = (size + PAGESIZE - 1) / PAGESIZE;
// Assume reserve() is for small objects.
Pool* pool = newPool(npages, false);
if (!pool)
return 0;
return pool.npages * PAGESIZE;
}
/**
* Update the thresholds for when to collect the next time
*/
void updateCollectThresholds() nothrow
{
static float max(float a, float b) nothrow
{
return a >= b ? a : b;
}
// instantly increases, slowly decreases
static float smoothDecay(float oldVal, float newVal) nothrow
{
// decay to 63.2% of newVal over 5 collections
// http://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter
enum alpha = 1.0 / (5 + 1);
immutable decay = (newVal - oldVal) * alpha + oldVal;
return max(newVal, decay);
}
immutable smTarget = usedSmallPages * config.heapSizeFactor;
smallCollectThreshold = smoothDecay(smallCollectThreshold, smTarget);
immutable lgTarget = usedLargePages * config.heapSizeFactor;
largeCollectThreshold = smoothDecay(largeCollectThreshold, lgTarget);
}
/**
* Minimizes physical memory usage by returning free pools to the OS.
*/
void minimize() nothrow
{
debug(PRINTF) printf("Minimizing.\n");
foreach (pool; pooltable.minimize())
{
debug(PRINTF) printFreeInfo(pool);
mappedPages -= pool.npages;
pool.Dtor();
cstdlib.free(pool);
}
debug(PRINTF) printf("Done minimizing.\n");
}
private @property bool lowMem() const nothrow
{
return isLowOnMem(mappedPages * PAGESIZE);
}
void* alloc(size_t size, ref size_t alloc_size, uint bits) nothrow
{
return size <= 2048 ? smallAlloc(binTable[size], alloc_size, bits)
: bigAlloc(size, alloc_size, bits);
}
void* smallAlloc(Bins bin, ref size_t alloc_size, uint bits) nothrow
{
alloc_size = binsize[bin];
void* p;
bool tryAlloc() nothrow
{
if (!bucket[bin])
{
bucket[bin] = allocPage(bin);
if (!bucket[bin])
return false;
}
p = bucket[bin];
return true;
}
if (!tryAlloc())
{
if (!lowMem && (disabled || usedSmallPages < smallCollectThreshold))
{
// disabled or threshold not reached => allocate a new pool instead of collecting
if (!newPool(1, false))
{
// out of memory => try to free some memory
fullcollect();
if (lowMem) minimize();
}
}
else
{
fullcollect();
if (lowMem) minimize();
}
// tryAlloc will succeed if a new pool was allocated above, if it fails allocate a new pool now
if (!tryAlloc() && (!newPool(1, false) || !tryAlloc()))
// out of luck or memory
onOutOfMemoryErrorNoGC();
}
assert(p !is null);
// Return next item from free list
bucket[bin] = (cast(List*)p).next;
auto pool = (cast(List*)p).pool;
if (bits)
pool.setBits((p - pool.baseAddr) >> pool.shiftBy, bits);
//debug(PRINTF) printf("\tmalloc => %p\n", p);
debug (MEMSTOMP) memset(p, 0xF0, alloc_size);
return p;
}
/**
* Allocate a chunk of memory that is larger than a page.
* Return null if out of memory.
*/
void* bigAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti = null) nothrow
{
debug(PRINTF) printf("In bigAlloc. Size: %d\n", size);
LargeObjectPool* pool;
size_t pn;
immutable npages = (size + PAGESIZE - 1) / PAGESIZE;
if (npages == 0)
onOutOfMemoryErrorNoGC(); // size just below size_t.max requested
bool tryAlloc() nothrow
{
foreach (p; pooltable[0 .. npools])
{
if (!p.isLargeObject || p.freepages < npages)
continue;
auto lpool = cast(LargeObjectPool*) p;
if ((pn = lpool.allocPages(npages)) == OPFAIL)
continue;
pool = lpool;
return true;
}
return false;
}
bool tryAllocNewPool() nothrow
{
pool = cast(LargeObjectPool*) newPool(npages, true);
if (!pool) return false;
pn = pool.allocPages(npages);
assert(pn != OPFAIL);
return true;
}
if (!tryAlloc())
{
if (!lowMem && (disabled || usedLargePages < largeCollectThreshold))
{
// disabled or threshold not reached => allocate a new pool instead of collecting
if (!tryAllocNewPool())
{
// disabled but out of memory => try to free some memory
fullcollect();
minimize();
}
}
else
{
fullcollect();
minimize();
}
// If alloc didn't yet succeed retry now that we collected/minimized
if (!pool && !tryAlloc() && !tryAllocNewPool())
// out of luck or memory
return null;
}
assert(pool);
debug(PRINTF) printFreeInfo(&pool.base);
pool.pagetable[pn] = B_PAGE;
if (npages > 1)
memset(&pool.pagetable[pn + 1], B_PAGEPLUS, npages - 1);
pool.updateOffsets(pn);
usedLargePages += npages;
pool.freepages -= npages;
debug(PRINTF) printFreeInfo(&pool.base);
auto p = pool.baseAddr + pn * PAGESIZE;
debug(PRINTF) printf("Got large alloc: %p, pt = %d, np = %d\n", p, pool.pagetable[pn], npages);
debug (MEMSTOMP) memset(p, 0xF1, size);
alloc_size = npages * PAGESIZE;
//debug(PRINTF) printf("\tp = %p\n", p);
if (bits)
pool.setBits(pn, bits);
return p;
}
/**
* Allocate a new pool with at least npages in it.
* Sort it into pooltable[].
* Return null if failed.
*/
Pool *newPool(size_t npages, bool isLargeObject) nothrow
{
//debug(PRINTF) printf("************Gcx::newPool(npages = %d)****************\n", npages);
// Minimum of POOLSIZE
size_t minPages = (config.minPoolSize << 20) / PAGESIZE;
if (npages < minPages)
npages = minPages;
else if (npages > minPages)
{ // Give us 150% of requested size, so there's room to extend
auto n = npages + (npages >> 1);
if (n < size_t.max/PAGESIZE)
npages = n;
}
// Allocate successively larger pools up to 8 megs
if (npools)
{ size_t n;
n = config.minPoolSize + config.incPoolSize * npools;
if (n > config.maxPoolSize)
n = config.maxPoolSize; // cap pool size
n *= (1 << 20) / PAGESIZE; // convert MB to pages
if (npages < n)
npages = n;
}
//printf("npages = %d\n", npages);
auto pool = cast(Pool *)cstdlib.calloc(1, isLargeObject ? LargeObjectPool.sizeof : SmallObjectPool.sizeof);
if (pool)
{
pool.initialize(npages, isLargeObject);
if (!pool.baseAddr || !pooltable.insert(pool))
{
pool.Dtor();
cstdlib.free(pool);
return null;
}
}
mappedPages += npages;
if (config.profile)
{
if (mappedPages * PAGESIZE > maxPoolMemory)
maxPoolMemory = mappedPages * PAGESIZE;
}
return pool;
}
/**
* Allocate a page of bin's.
* Returns:
* head of a single linked list of new entries
*/
List* allocPage(Bins bin) nothrow
{
//debug(PRINTF) printf("Gcx::allocPage(bin = %d)\n", bin);
for (size_t n = 0; n < npools; n++)
{
Pool* pool = pooltable[n];
if (pool.isLargeObject)
continue;
if (List* p = (cast(SmallObjectPool*)pool).allocPage(bin))
{
++usedSmallPages;
return p;
}
}
return null;
}
static struct ToScanStack
{
nothrow:
@disable this(this);
void reset()
{
_length = 0;
os_mem_unmap(_p, _cap * Range.sizeof);
_p = null;
_cap = 0;
}
void push(Range rng)
{
if (_length == _cap) grow();
_p[_length++] = rng;
}
Range pop()
in { assert(!empty); }
body
{
return _p[--_length];
}
ref inout(Range) opIndex(size_t idx) inout
in { assert(idx < _length); }
body
{
return _p[idx];
}
@property size_t length() const { return _length; }
@property bool empty() const { return !length; }
private:
void grow()
{
enum initSize = 64 * 1024; // Windows VirtualAlloc granularity
immutable ncap = _cap ? 2 * _cap : initSize / Range.sizeof;
auto p = cast(Range*)os_mem_map(ncap * Range.sizeof);
if (p is null) onOutOfMemoryErrorNoGC();
if (_p !is null)
{
p[0 .. _length] = _p[0 .. _length];
os_mem_unmap(_p, _cap * Range.sizeof);
}
_p = p;
_cap = ncap;
}
size_t _length;
Range* _p;
size_t _cap;
}
ToScanStack toscan;
/**
* Search a range of memory values and mark any pointers into the GC pool.
*/
void mark(void *pbot, void *ptop) scope nothrow
{
void **p1 = cast(void **)pbot;
void **p2 = cast(void **)ptop;
// limit the amount of ranges added to the toscan stack
enum FANOUT_LIMIT = 32;
size_t stackPos;
Range[FANOUT_LIMIT] stack = void;
Lagain:
size_t pcache = 0;
// let dmd allocate a register for this.pools
auto pools = pooltable.pools;
const highpool = pooltable.npools - 1;
const minAddr = pooltable.minAddr;
const maxAddr = pooltable.maxAddr;
//printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
Lnext: for (; p1 < p2; p1++)
{
auto p = *p1;
//if (log) debug(PRINTF) printf("\tmark %p\n", p);
if (p >= minAddr && p < maxAddr)
{
if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache)
continue;
Pool* pool = void;
size_t low = 0;
size_t high = highpool;
while (true)
{
size_t mid = (low + high) >> 1;
pool = pools[mid];
if (p < pool.baseAddr)
high = mid - 1;
else if (p >= pool.topAddr)
low = mid + 1;
else break;
if (low > high)
continue Lnext;
}
size_t offset = cast(size_t)(p - pool.baseAddr);
size_t biti = void;
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pool.pagetable[pn];
void* base = void;
//debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti);
// Adjust bit to be at start of allocated memory block
if (bin < B_PAGE)
{
// We don't care abou setting pointsToBase correctly
// because it's ignored for small object pools anyhow.
auto offsetBase = offset & notbinsize[bin];
biti = offsetBase >> pool.shiftBy;
base = pool.baseAddr + offsetBase;
//debug(PRINTF) printf("\t\tbiti = x%x\n", biti);
if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + binsize[bin]);
if (stackPos == stack.length)
break;
}
}
else if (bin == B_PAGE)
{
auto offsetBase = offset & notbinsize[bin];
base = pool.baseAddr + offsetBase;
biti = offsetBase >> pool.shiftBy;
//debug(PRINTF) printf("\t\tbiti = x%x\n", biti);
pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1);
// For the NO_INTERIOR attribute. This tracks whether
// the pointer is an interior pointer or points to the
// base address of a block.
bool pointsToBase = (base == sentinel_sub(p));
if (!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti))
continue;
if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE);
if (stackPos == stack.length)
break;
}
}
else if (bin == B_PAGEPLUS)
{
pn -= pool.bPageOffsets[pn];
base = pool.baseAddr + (pn * PAGESIZE);
biti = pn * (PAGESIZE >> pool.shiftBy);
pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1);
if (pool.nointerior.nbits && pool.nointerior.test(biti))
continue;
if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE);
if (stackPos == stack.length)
break;
}
}
else
{
// Don't mark bits in B_FREE pages
assert(bin == B_FREE);
continue;
}
}
}
Range next=void;
if (p1 < p2)
{
// local stack is full, push it to the global stack
assert(stackPos == stack.length);
toscan.push(Range(p1, p2));
// reverse order for depth-first-order traversal
foreach_reverse (ref rng; stack[0 .. $ - 1])
toscan.push(rng);
stackPos = 0;
next = stack[$-1];
}
else if (stackPos)
{
// pop range from local stack and recurse
next = stack[--stackPos];
}
else if (!toscan.empty)
{
// pop range from global stack and recurse
next = toscan.pop();
}
else
{
// nothing more to do
return;
}
p1 = cast(void**)next.pbot;
p2 = cast(void**)next.ptop;
// printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
goto Lagain;
}
// collection step 1: prepare freebits and mark bits
void prepare() nothrow
{
size_t n;
Pool* pool;
for (n = 0; n < npools; n++)
{
pool = pooltable[n];
pool.mark.zero();
if (!pool.isLargeObject) pool.freebits.zero();
}
debug(COLLECT_PRINTF) printf("Set bits\n");
// Mark each free entry, so it doesn't get scanned
for (n = 0; n < B_PAGE; n++)
{
for (List *list = bucket[n]; list; list = list.next)
{
pool = list.pool;
assert(pool);
pool.freebits.set(cast(size_t)(cast(void*)list - pool.baseAddr) / 16);
}
}
debug(COLLECT_PRINTF) printf("Marked free entries.\n");
for (n = 0; n < npools; n++)
{
pool = pooltable[n];
if (!pool.isLargeObject)
{
pool.mark.copy(&pool.freebits);
}
}
}
// collection step 2: mark roots and heap
void markAll(bool nostack) nothrow
{
if (!nostack)
{
debug(COLLECT_PRINTF) printf("\tscan stacks.\n");
// Scan stacks and registers for each paused thread
thread_scanAll(&mark);
}
// Scan roots[]
debug(COLLECT_PRINTF) printf("\tscan roots[]\n");
foreach (root; roots)
{
mark(cast(void*)&root.proot, cast(void*)(&root.proot + 1));
}
// Scan ranges[]
debug(COLLECT_PRINTF) printf("\tscan ranges[]\n");
//log++;
foreach (range; ranges)
{
debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop);
mark(range.pbot, range.ptop);
}
//log--;
}
// collection step 3: free all unreferenced objects
size_t sweep() nothrow
{
// Free up everything not marked
debug(COLLECT_PRINTF) printf("\tfree'ing\n");
size_t freedLargePages;
size_t freed;
for (size_t n = 0; n < npools; n++)
{
size_t pn;
Pool* pool = pooltable[n];
if (pool.isLargeObject)
{
for (pn = 0; pn < pool.npages; pn++)
{
Bins bin = cast(Bins)pool.pagetable[pn];
if (bin > B_PAGE) continue;
size_t biti = pn;
if (!pool.mark.test(biti))
{
void *p = pool.baseAddr + pn * PAGESIZE;
void* q = sentinel_add(p);
sentinel_Invariant(q);
if (pool.finals.nbits && pool.finals.clear(biti))
{
size_t size = pool.bPageOffsets[pn] * PAGESIZE - SENTINEL_EXTRA;
uint attr = pool.getBits(biti);
rt_finalizeFromGC(q, size, attr);
}
pool.clrBits(biti, ~BlkAttr.NONE ^ BlkAttr.FINALIZE);
debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p);
log_free(q);
pool.pagetable[pn] = B_FREE;
if (pn < pool.searchStart) pool.searchStart = pn;
freedLargePages++;
pool.freepages++;
debug (MEMSTOMP) memset(p, 0xF3, PAGESIZE);
while (pn + 1 < pool.npages && pool.pagetable[pn + 1] == B_PAGEPLUS)
{
pn++;
pool.pagetable[pn] = B_FREE;
// Don't need to update searchStart here because
// pn is guaranteed to be greater than last time
// we updated it.
pool.freepages++;
freedLargePages++;
debug (MEMSTOMP)
{ p += PAGESIZE;
memset(p, 0xF3, PAGESIZE);
}
}
pool.largestFree = pool.freepages; // invalidate
}
}
}
else
{
for (pn = 0; pn < pool.npages; pn++)
{
Bins bin = cast(Bins)pool.pagetable[pn];
if (bin < B_PAGE)
{
immutable size = binsize[bin];
void *p = pool.baseAddr + pn * PAGESIZE;
void *ptop = p + PAGESIZE;
immutable base = pn * (PAGESIZE/16);
immutable bitstride = size / 16;
bool freeBits;
PageBits toFree;
for (size_t i; p < ptop; p += size, i += bitstride)
{
immutable biti = base + i;
if (!pool.mark.test(biti))
{
void* q = sentinel_add(p);
sentinel_Invariant(q);
if (pool.finals.nbits && pool.finals.test(biti))
rt_finalizeFromGC(q, size - SENTINEL_EXTRA, pool.getBits(biti));
freeBits = true;
toFree.set(i);
debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p);
log_free(sentinel_add(p));
debug (MEMSTOMP) memset(p, 0xF3, size);
freed += size;
}
}
if (freeBits)
pool.freePageBits(pn, toFree);
}
}
}
}
assert(freedLargePages <= usedLargePages);
usedLargePages -= freedLargePages;
debug(COLLECT_PRINTF) printf("\tfree'd %u bytes, %u pages from %u pools\n", freed, freedLargePages, npools);
return freedLargePages;
}
// collection step 4: recover pages with no live objects, rebuild free lists
size_t recover() nothrow
{
// init tail list
List**[B_PAGE] tail = void;
foreach (i, ref next; tail)
next = &bucket[i];
// Free complete pages, rebuild free list
debug(COLLECT_PRINTF) printf("\tfree complete pages\n");
size_t freedSmallPages;
for (size_t n = 0; n < npools; n++)
{
size_t pn;
Pool* pool = pooltable[n];
if (pool.isLargeObject)
continue;
for (pn = 0; pn < pool.npages; pn++)
{
Bins bin = cast(Bins)pool.pagetable[pn];
size_t biti;
size_t u;
if (bin < B_PAGE)
{
size_t size = binsize[bin];
size_t bitstride = size / 16;
size_t bitbase = pn * (PAGESIZE / 16);
size_t bittop = bitbase + (PAGESIZE / 16);
void* p;
biti = bitbase;
for (biti = bitbase; biti < bittop; biti += bitstride)
{
if (!pool.freebits.test(biti))
goto Lnotfree;
}
pool.pagetable[pn] = B_FREE;
if (pn < pool.searchStart) pool.searchStart = pn;
pool.freepages++;
freedSmallPages++;
continue;
Lnotfree:
p = pool.baseAddr + pn * PAGESIZE;
for (u = 0; u < PAGESIZE; u += size)
{
biti = bitbase + u / 16;
if (!pool.freebits.test(biti))
continue;
auto elem = cast(List *)(p + u);
elem.pool = pool;
*tail[bin] = elem;
tail[bin] = &elem.next;
}
}
}
}
// terminate tail list
foreach (ref next; tail)
*next = null;
assert(freedSmallPages <= usedSmallPages);
usedSmallPages -= freedSmallPages;
debug(COLLECT_PRINTF) printf("\trecovered pages = %d\n", freedSmallPages);
return freedSmallPages;
}
/**
* Return number of full pages free'd.
*/
size_t fullcollect(bool nostack = false) nothrow
{
MonoTime start, stop, begin;
if (config.profile)
{
begin = start = currTime;
}
debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n");
//printf("\tpool address range = %p .. %p\n", minAddr, maxAddr);
{
// lock roots and ranges around suspending threads b/c they're not reentrant safe
rangesLock.lock();
rootsLock.lock();
scope (exit)
{
rangesLock.unlock();
rootsLock.unlock();
}
thread_suspendAll();
prepare();
if (config.profile)
{
stop = currTime;
prepTime += (stop - start);
start = stop;
}
markAll(nostack);
thread_processGCMarks(&isMarked);
thread_resumeAll();
}
if (config.profile)
{
stop = currTime;
markTime += (stop - start);
Duration pause = stop - begin;
if (pause > maxPauseTime)
maxPauseTime = pause;
start = stop;
}
ConservativeGC._inFinalizer = true;
size_t freedLargePages=void;
{
scope (failure) ConservativeGC._inFinalizer = false;
freedLargePages = sweep();
ConservativeGC._inFinalizer = false;
}
if (config.profile)
{
stop = currTime;
sweepTime += (stop - start);
start = stop;
}
immutable freedSmallPages = recover();
if (config.profile)
{
stop = currTime;
recoverTime += (stop - start);
++numCollections;
}
updateCollectThresholds();
return freedLargePages + freedSmallPages;
}
/**
* Returns true if the addr lies within a marked block.
*
* Warning! This should only be called while the world is stopped inside
* the fullcollect function.
*/
int isMarked(void *addr) scope nothrow
{
// first, we find the Pool this block is in, then check to see if the
// mark bit is clear.
auto pool = findPool(addr);
if (pool)
{
auto offset = cast(size_t)(addr - pool.baseAddr);
auto pn = offset / PAGESIZE;
auto bins = cast(Bins)pool.pagetable[pn];
size_t biti = void;
if (bins <= B_PAGE)
{
biti = (offset & notbinsize[bins]) >> pool.shiftBy;
}
else if (bins == B_PAGEPLUS)
{
pn -= pool.bPageOffsets[pn];
biti = pn * (PAGESIZE >> pool.shiftBy);
}
else // bins == B_FREE
{
assert(bins == B_FREE);
return IsMarked.no;
}
return pool.mark.test(biti) ? IsMarked.yes : IsMarked.no;
}
return IsMarked.unknown;
}
/***** Leak Detector ******/
debug (LOGGING)
{
LogArray current;
LogArray prev;
void log_init()
{
//debug(PRINTF) printf("+log_init()\n");
current.reserve(1000);
prev.reserve(1000);
//debug(PRINTF) printf("-log_init()\n");
}
void log_malloc(void *p, size_t size) nothrow
{
//debug(PRINTF) printf("+log_malloc(p = %p, size = %zd)\n", p, size);
Log log;
log.p = p;
log.size = size;
log.line = GC.line;
log.file = GC.file;
log.parent = null;
GC.line = 0;
GC.file = null;
current.push(log);
//debug(PRINTF) printf("-log_malloc()\n");
}
void log_free(void *p) nothrow
{
//debug(PRINTF) printf("+log_free(%p)\n", p);
auto i = current.find(p);
if (i == OPFAIL)
{
debug(PRINTF) printf("free'ing unallocated memory %p\n", p);
}
else
current.remove(i);
//debug(PRINTF) printf("-log_free()\n");
}
void log_collect() nothrow
{
//debug(PRINTF) printf("+log_collect()\n");
// Print everything in current that is not in prev
debug(PRINTF) printf("New pointers this cycle: --------------------------------\n");
size_t used = 0;
for (size_t i = 0; i < current.dim; i++)
{
auto j = prev.find(current.data[i].p);
if (j == OPFAIL)
current.data[i].print();
else
used++;
}
debug(PRINTF) printf("All roots this cycle: --------------------------------\n");
for (size_t i = 0; i < current.dim; i++)
{
void* p = current.data[i].p;
if (!findPool(current.data[i].parent))
{
auto j = prev.find(current.data[i].p);
debug(PRINTF) printf(j == OPFAIL ? "N" : " ");
current.data[i].print();
}
}
debug(PRINTF) printf("Used = %d-------------------------------------------------\n", used);
prev.copy(&current);
debug(PRINTF) printf("-log_collect()\n");
}
void log_parent(void *p, void *parent) nothrow
{
//debug(PRINTF) printf("+log_parent()\n");
auto i = current.find(p);
if (i == OPFAIL)
{
debug(PRINTF) printf("parent'ing unallocated memory %p, parent = %p\n", p, parent);
Pool *pool;
pool = findPool(p);
assert(pool);
size_t offset = cast(size_t)(p - pool.baseAddr);
size_t biti;
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pool.pagetable[pn];
biti = (offset & notbinsize[bin]);
debug(PRINTF) printf("\tbin = %d, offset = x%x, biti = x%x\n", bin, offset, biti);
}
else
{
current.data[i].parent = parent;
}
//debug(PRINTF) printf("-log_parent()\n");
}
}
else
{
void log_init() nothrow { }
void log_malloc(void *p, size_t size) nothrow { }
void log_free(void *p) nothrow { }
void log_collect() nothrow { }
void log_parent(void *p, void *parent) nothrow { }
}
}
/* ============================ Pool =============================== */
struct Pool
{
void* baseAddr;
void* topAddr;
GCBits mark; // entries already scanned, or should not be scanned
GCBits freebits; // entries that are on the free list
GCBits finals; // entries that need finalizer run on them
GCBits structFinals;// struct entries that need a finalzier run on them
GCBits noscan; // entries that should not be scanned
GCBits appendable; // entries that are appendable
GCBits nointerior; // interior pointers should be ignored.
// Only implemented for large object pools.
size_t npages;
size_t freepages; // The number of pages not in use.
ubyte* pagetable;
bool isLargeObject;
uint shiftBy; // shift count for the divisor used for determining bit indices.
// This tracks how far back we have to go to find the nearest B_PAGE at
// a smaller address than a B_PAGEPLUS. To save space, we use a uint.
// This limits individual allocations to 16 terabytes, assuming a 4k
// pagesize.
uint* bPageOffsets;
// This variable tracks a conservative estimate of where the first free
// page in this pool is, so that if a lot of pages towards the beginning
// are occupied, we can bypass them in O(1).
size_t searchStart;
size_t largestFree; // upper limit for largest free chunk in large object pool
void initialize(size_t npages, bool isLargeObject) nothrow
{
this.isLargeObject = isLargeObject;
size_t poolsize;
shiftBy = isLargeObject ? 12 : 4;
//debug(PRINTF) printf("Pool::Pool(%u)\n", npages);
poolsize = npages * PAGESIZE;
assert(poolsize >= POOLSIZE);
baseAddr = cast(byte *)os_mem_map(poolsize);
// Some of the code depends on page alignment of memory pools
assert((cast(size_t)baseAddr & (PAGESIZE - 1)) == 0);
if (!baseAddr)
{
//debug(PRINTF) printf("GC fail: poolsize = x%zx, errno = %d\n", poolsize, errno);
//debug(PRINTF) printf("message = '%s'\n", sys_errlist[errno]);
npages = 0;
poolsize = 0;
}
//assert(baseAddr);
topAddr = baseAddr + poolsize;
auto nbits = cast(size_t)poolsize >> shiftBy;
mark.alloc(nbits);
// pagetable already keeps track of what's free for the large object
// pool.
if (!isLargeObject)
{
freebits.alloc(nbits);
}
noscan.alloc(nbits);
appendable.alloc(nbits);
pagetable = cast(ubyte*)cstdlib.malloc(npages);
if (!pagetable)
onOutOfMemoryErrorNoGC();
if (isLargeObject)
{
bPageOffsets = cast(uint*)cstdlib.malloc(npages * uint.sizeof);
if (!bPageOffsets)
onOutOfMemoryErrorNoGC();
}
memset(pagetable, B_FREE, npages);
this.npages = npages;
this.freepages = npages;
this.searchStart = 0;
this.largestFree = npages;
}
void Dtor() nothrow
{
if (baseAddr)
{
int result;
if (npages)
{
result = os_mem_unmap(baseAddr, npages * PAGESIZE);
assert(result == 0);
npages = 0;
}
baseAddr = null;
topAddr = null;
}
if (pagetable)
{
cstdlib.free(pagetable);
pagetable = null;
}
if (bPageOffsets)
cstdlib.free(bPageOffsets);
mark.Dtor();
if (isLargeObject)
{
nointerior.Dtor();
}
else
{
freebits.Dtor();
}
finals.Dtor();
structFinals.Dtor();
noscan.Dtor();
appendable.Dtor();
}
/**
*
*/
uint getBits(size_t biti) nothrow
{
uint bits;
if (finals.nbits && finals.test(biti))
bits |= BlkAttr.FINALIZE;
if (structFinals.nbits && structFinals.test(biti))
bits |= BlkAttr.STRUCTFINAL;
if (noscan.test(biti))
bits |= BlkAttr.NO_SCAN;
if (nointerior.nbits && nointerior.test(biti))
bits |= BlkAttr.NO_INTERIOR;
if (appendable.test(biti))
bits |= BlkAttr.APPENDABLE;
return bits;
}
/**
*
*/
void clrBits(size_t biti, uint mask) nothrow
{
immutable dataIndex = biti >> GCBits.BITS_SHIFT;
immutable bitOffset = biti & GCBits.BITS_MASK;
immutable keep = ~(GCBits.BITS_1 << bitOffset);
if (mask & BlkAttr.FINALIZE && finals.nbits)
finals.data[dataIndex] &= keep;
if (structFinals.nbits && (mask & BlkAttr.STRUCTFINAL))
structFinals.data[dataIndex] &= keep;
if (mask & BlkAttr.NO_SCAN)
noscan.data[dataIndex] &= keep;
if (mask & BlkAttr.APPENDABLE)
appendable.data[dataIndex] &= keep;
if (nointerior.nbits && (mask & BlkAttr.NO_INTERIOR))
nointerior.data[dataIndex] &= keep;
}
/**
*
*/
void setBits(size_t biti, uint mask) nothrow
{
// Calculate the mask and bit offset once and then use it to
// set all of the bits we need to set.
immutable dataIndex = biti >> GCBits.BITS_SHIFT;
immutable bitOffset = biti & GCBits.BITS_MASK;
immutable orWith = GCBits.BITS_1 << bitOffset;
if (mask & BlkAttr.STRUCTFINAL)
{
if (!structFinals.nbits)
structFinals.alloc(mark.nbits);
structFinals.data[dataIndex] |= orWith;
}
if (mask & BlkAttr.FINALIZE)
{
if (!finals.nbits)
finals.alloc(mark.nbits);
finals.data[dataIndex] |= orWith;
}
if (mask & BlkAttr.NO_SCAN)
{
noscan.data[dataIndex] |= orWith;
}
// if (mask & BlkAttr.NO_MOVE)
// {
// if (!nomove.nbits)
// nomove.alloc(mark.nbits);
// nomove.data[dataIndex] |= orWith;
// }
if (mask & BlkAttr.APPENDABLE)
{
appendable.data[dataIndex] |= orWith;
}
if (isLargeObject && (mask & BlkAttr.NO_INTERIOR))
{
if (!nointerior.nbits)
nointerior.alloc(mark.nbits);
nointerior.data[dataIndex] |= orWith;
}
}
void freePageBits(size_t pagenum, in ref PageBits toFree) nothrow
{
assert(!isLargeObject);
assert(!nointerior.nbits); // only for large objects
import core.internal.traits : staticIota;
immutable beg = pagenum * (PAGESIZE / 16 / GCBits.BITS_PER_WORD);
foreach (i; staticIota!(0, PageBits.length))
{
immutable w = toFree[i];
if (!w) continue;
immutable wi = beg + i;
freebits.data[wi] |= w;
noscan.data[wi] &= ~w;
appendable.data[wi] &= ~w;
}
if (finals.nbits)
{
foreach (i; staticIota!(0, PageBits.length))
if (toFree[i])
finals.data[beg + i] &= ~toFree[i];
}
if (structFinals.nbits)
{
foreach (i; staticIota!(0, PageBits.length))
if (toFree[i])
structFinals.data[beg + i] &= ~toFree[i];
}
}
/**
* Given a pointer p in the p, return the pagenum.
*/
size_t pagenumOf(void *p) const nothrow
in
{
assert(p >= baseAddr);
assert(p < topAddr);
}
body
{
return cast(size_t)(p - baseAddr) / PAGESIZE;
}
@property bool isFree() const pure nothrow
{
return npages == freepages;
}
size_t slGetSize(void* p) nothrow
{
if (isLargeObject)
return (cast(LargeObjectPool*)&this).getSize(p);
else
return (cast(SmallObjectPool*)&this).getSize(p);
}
BlkInfo slGetInfo(void* p) nothrow
{
if (isLargeObject)
return (cast(LargeObjectPool*)&this).getInfo(p);
else
return (cast(SmallObjectPool*)&this).getInfo(p);
}
void Invariant() const {}
debug(INVARIANT)
invariant()
{
//mark.Invariant();
//scan.Invariant();
//freebits.Invariant();
//finals.Invariant();
//structFinals.Invariant();
//noscan.Invariant();
//appendable.Invariant();
//nointerior.Invariant();
if (baseAddr)
{
//if (baseAddr + npages * PAGESIZE != topAddr)
//printf("baseAddr = %p, npages = %d, topAddr = %p\n", baseAddr, npages, topAddr);
assert(baseAddr + npages * PAGESIZE == topAddr);
}
if (pagetable !is null)
{
for (size_t i = 0; i < npages; i++)
{
Bins bin = cast(Bins)pagetable[i];
assert(bin < B_MAX);
}
}
}
}
struct LargeObjectPool
{
Pool base;
alias base this;
void updateOffsets(size_t fromWhere) nothrow
{
assert(pagetable[fromWhere] == B_PAGE);
size_t pn = fromWhere + 1;
for (uint offset = 1; pn < npages; pn++, offset++)
{
if (pagetable[pn] != B_PAGEPLUS) break;
bPageOffsets[pn] = offset;
}
// Store the size of the block in bPageOffsets[fromWhere].
bPageOffsets[fromWhere] = cast(uint) (pn - fromWhere);
}
/**
* Allocate n pages from Pool.
* Returns OPFAIL on failure.
*/
size_t allocPages(size_t n) nothrow
{
if (largestFree < n || searchStart + n > npages)
return OPFAIL;
//debug(PRINTF) printf("Pool::allocPages(n = %d)\n", n);
size_t largest = 0;
if (pagetable[searchStart] == B_PAGEPLUS)
{
searchStart -= bPageOffsets[searchStart]; // jump to B_PAGE
searchStart += bPageOffsets[searchStart];
}
while (searchStart < npages && pagetable[searchStart] == B_PAGE)
searchStart += bPageOffsets[searchStart];
for (size_t i = searchStart; i < npages; )
{
assert(pagetable[i] == B_FREE);
size_t p = 1;
while (p < n && i + p < npages && pagetable[i + p] == B_FREE)
p++;
if (p == n)
return i;
if (p > largest)
largest = p;
i += p;
while (i < npages && pagetable[i] == B_PAGE)
{
// we have the size information, so we skip a whole bunch of pages.
i += bPageOffsets[i];
}
}
// not enough free pages found, remember largest free chunk
largestFree = largest;
return OPFAIL;
}
/**
* Free npages pages starting with pagenum.
*/
void freePages(size_t pagenum, size_t npages) nothrow
{
//memset(&pagetable[pagenum], B_FREE, npages);
if (pagenum < searchStart)
searchStart = pagenum;
for (size_t i = pagenum; i < npages + pagenum; i++)
{
if (pagetable[i] < B_FREE)
{
freepages++;
}
pagetable[i] = B_FREE;
}
largestFree = freepages; // invalidate
}
/**
* Get size of pointer p in pool.
*/
size_t getSize(void *p) const nothrow
in
{
assert(p >= baseAddr);
assert(p < topAddr);
}
body
{
size_t pagenum = pagenumOf(p);
Bins bin = cast(Bins)pagetable[pagenum];
assert(bin == B_PAGE);
return bPageOffsets[pagenum] * PAGESIZE;
}
/**
*
*/
BlkInfo getInfo(void* p) nothrow
{
BlkInfo info;
size_t offset = cast(size_t)(p - baseAddr);
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pagetable[pn];
if (bin == B_PAGEPLUS)
pn -= bPageOffsets[pn];
else if (bin != B_PAGE)
return info; // no info for free pages
info.base = baseAddr + pn * PAGESIZE;
info.size = bPageOffsets[pn] * PAGESIZE;
info.attr = getBits(pn);
return info;
}
void runFinalizers(in void[] segment) nothrow
{
foreach (pn; 0 .. npages)
{
Bins bin = cast(Bins)pagetable[pn];
if (bin > B_PAGE)
continue;
size_t biti = pn;
if (!finals.test(biti))
continue;
auto p = sentinel_add(baseAddr + pn * PAGESIZE);
size_t size = bPageOffsets[pn] * PAGESIZE - SENTINEL_EXTRA;
uint attr = getBits(biti);
if (!rt_hasFinalizerInSegment(p, size, attr, segment))
continue;
rt_finalizeFromGC(p, size, attr);
clrBits(biti, ~BlkAttr.NONE);
if (pn < searchStart)
searchStart = pn;
debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p);
//log_free(sentinel_add(p));
size_t n = 1;
for (; pn + n < npages; ++n)
if (pagetable[pn + n] != B_PAGEPLUS)
break;
debug (MEMSTOMP) memset(baseAddr + pn * PAGESIZE, 0xF3, n * PAGESIZE);
freePages(pn, n);
}
}
}
struct SmallObjectPool
{
Pool base;
alias base this;
/**
* Get size of pointer p in pool.
*/
size_t getSize(void *p) const nothrow
in
{
assert(p >= baseAddr);
assert(p < topAddr);
}
body
{
size_t pagenum = pagenumOf(p);
Bins bin = cast(Bins)pagetable[pagenum];
assert(bin < B_PAGE);
return binsize[bin];
}
BlkInfo getInfo(void* p) nothrow
{
BlkInfo info;
size_t offset = cast(size_t)(p - baseAddr);
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pagetable[pn];
if (bin >= B_PAGE)
return info;
info.base = cast(void*)((cast(size_t)p) & notbinsize[bin]);
info.size = binsize[bin];
offset = info.base - baseAddr;
info.attr = getBits(cast(size_t)(offset >> shiftBy));
return info;
}
void runFinalizers(in void[] segment) nothrow
{
foreach (pn; 0 .. npages)
{
Bins bin = cast(Bins)pagetable[pn];
if (bin >= B_PAGE)
continue;
immutable size = binsize[bin];
auto p = baseAddr + pn * PAGESIZE;
const ptop = p + PAGESIZE;
immutable base = pn * (PAGESIZE/16);
immutable bitstride = size / 16;
bool freeBits;
PageBits toFree;
for (size_t i; p < ptop; p += size, i += bitstride)
{
immutable biti = base + i;
if (!finals.test(biti))
continue;
auto q = sentinel_add(p);
uint attr = getBits(biti);
if (!rt_hasFinalizerInSegment(q, size, attr, segment))
continue;
rt_finalizeFromGC(q, size, attr);
freeBits = true;
toFree.set(i);
debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p);
//log_free(sentinel_add(p));
debug (MEMSTOMP) memset(p, 0xF3, size);
}
if (freeBits)
freePageBits(pn, toFree);
}
}
/**
* Allocate a page of bin's.
* Returns:
* head of a single linked list of new entries
*/
List* allocPage(Bins bin) nothrow
{
size_t pn;
for (pn = searchStart; pn < npages; pn++)
if (pagetable[pn] == B_FREE)
goto L1;
return null;
L1:
searchStart = pn + 1;
pagetable[pn] = cast(ubyte)bin;
freepages--;
// Convert page to free list
size_t size = binsize[bin];
void* p = baseAddr + pn * PAGESIZE;
void* ptop = p + PAGESIZE - size;
auto first = cast(List*) p;
for (; p < ptop; p += size)
{
(cast(List *)p).next = cast(List *)(p + size);
(cast(List *)p).pool = &base;
}
(cast(List *)p).next = null;
(cast(List *)p).pool = &base;
return first;
}
}
unittest // bugzilla 14467
{
int[] arr = new int[10];
assert(arr.capacity);
arr = arr[$..$];
assert(arr.capacity);
}
unittest // bugzilla 15353
{
import core.memory : GC;
static struct Foo
{
~this()
{
GC.free(buf); // ignored in finalizer
}
void* buf;
}
new Foo(GC.malloc(10));
GC.collect();
}
unittest // bugzilla 15822
{
import core.memory : GC;
ubyte[16] buf;
static struct Foo
{
~this()
{
GC.removeRange(ptr);
GC.removeRoot(ptr);
}
ubyte* ptr;
}
GC.addRoot(buf.ptr);
GC.addRange(buf.ptr, buf.length);
new Foo(buf.ptr);
GC.collect();
}
unittest // bugzilla 1180
{
import core.exception;
try
{
size_t x = size_t.max - 100;
byte[] big_buf = new byte[x];
}
catch (OutOfMemoryError)
{
}
}
/* ============================ SENTINEL =============================== */
debug (SENTINEL)
{
const size_t SENTINEL_PRE = cast(size_t) 0xF4F4F4F4F4F4F4F4UL; // 32 or 64 bits
const ubyte SENTINEL_POST = 0xF5; // 8 bits
const uint SENTINEL_EXTRA = 2 * size_t.sizeof + 1;
inout(size_t*) sentinel_size(inout void *p) nothrow { return &(cast(inout size_t *)p)[-2]; }
inout(size_t*) sentinel_pre(inout void *p) nothrow { return &(cast(inout size_t *)p)[-1]; }
inout(ubyte*) sentinel_post(inout void *p) nothrow { return &(cast(inout ubyte *)p)[*sentinel_size(p)]; }
void sentinel_init(void *p, size_t size) nothrow
{
*sentinel_size(p) = size;
*sentinel_pre(p) = SENTINEL_PRE;
*sentinel_post(p) = SENTINEL_POST;
}
void sentinel_Invariant(const void *p) nothrow
{
debug
{
assert(*sentinel_pre(p) == SENTINEL_PRE);
assert(*sentinel_post(p) == SENTINEL_POST);
}
else if (*sentinel_pre(p) != SENTINEL_PRE || *sentinel_post(p) != SENTINEL_POST)
onInvalidMemoryOperationError(); // also trigger in release build
}
void *sentinel_add(void *p) nothrow
{
return p + 2 * size_t.sizeof;
}
void *sentinel_sub(void *p) nothrow
{
return p - 2 * size_t.sizeof;
}
}
else
{
const uint SENTINEL_EXTRA = 0;
void sentinel_init(void *p, size_t size) nothrow
{
}
void sentinel_Invariant(const void *p) nothrow
{
}
void *sentinel_add(void *p) nothrow
{
return p;
}
void *sentinel_sub(void *p) nothrow
{
return p;
}
}
debug (MEMSTOMP)
unittest
{
import core.memory;
auto p = cast(uint*)GC.malloc(uint.sizeof*3);
assert(*p == 0xF0F0F0F0);
p[2] = 0; // First two will be used for free list
GC.free(p);
assert(p[2] == 0xF2F2F2F2);
}
debug (SENTINEL)
unittest
{
import core.memory;
auto p = cast(ubyte*)GC.malloc(1);
assert(p[-1] == 0xF4);
assert(p[ 1] == 0xF5);
/*
p[1] = 0;
bool thrown;
try
GC.free(p);
catch (Error e)
thrown = true;
p[1] = 0xF5;
assert(thrown);
*/
}
unittest
{
import core.memory;
// https://issues.dlang.org/show_bug.cgi?id=9275
GC.removeRoot(null);
GC.removeRoot(cast(void*)13);
}
// improve predictability of coverage of code that is eventually not hit by other tests
unittest
{
import core.memory;
auto p = GC.malloc(260 << 20); // new pool has 390 MB
auto q = GC.malloc(65 << 20); // next chunk (larger than 64MB to ensure the same pool is used)
auto r = GC.malloc(65 << 20); // another chunk in same pool
assert(p + (260 << 20) == q);
assert(q + (65 << 20) == r);
GC.free(q);
// should trigger "assert(bin == B_FREE);" in mark due to dangling pointer q:
GC.collect();
// should trigger "break;" in extendNoSync:
size_t sz = GC.extend(p, 64 << 20, 66 << 20); // trigger size after p large enough (but limited)
assert(sz == 325 << 20);
GC.free(p);
GC.free(r);
r = q; // ensure q is not trashed before collection above
p = GC.malloc(70 << 20); // from the same pool
q = GC.malloc(70 << 20);
r = GC.malloc(70 << 20);
auto s = GC.malloc(70 << 20);
auto t = GC.malloc(70 << 20); // 350 MB of 390 MB used
assert(p + (70 << 20) == q);
assert(q + (70 << 20) == r);
assert(r + (70 << 20) == s);
assert(s + (70 << 20) == t);
GC.free(r); // ensure recalculation of largestFree in nxxt allocPages
auto z = GC.malloc(75 << 20); // needs new pool
GC.free(p);
GC.free(q);
GC.free(s);
GC.free(t);
GC.free(z);
GC.minimize(); // release huge pool
}