2010-12-03 05:34:57 +01:00
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
2011-03-17 00:05:44 +01:00
// Garbage collector.
2010-12-03 05:34:57 +01:00
2012-04-20 06:58:26 +02:00
# include <unistd.h>
2010-12-03 05:34:57 +01:00
# include "runtime.h"
2011-10-27 01:57:58 +02:00
# include "arch.h"
2010-12-03 05:34:57 +01:00
# include "malloc.h"
2012-12-22 02:15:33 +01:00
# include "mgc0.h"
2012-10-23 06:31:11 +02:00
# include "race.h"
2013-01-29 21:52:43 +01:00
# include "go-type.h"
// Map gccgo field names to gc field names.
// Slice aka __go_open_array.
# define array __values
# define cap __capacity
// Iface aka __go_interface
# define tab __methods
// Eface aka __go_empty_interface.
# define type __type_descriptor
2013-07-16 08:54:42 +02:00
// Hmap aka __go_map
typedef struct __go_map Hmap ;
2013-01-29 21:52:43 +01:00
// Type aka __go_type_descriptor
# define kind __code
2013-07-16 08:54:42 +02:00
# define string __reflection
2013-01-29 21:52:43 +01:00
# define KindPtr GO_PTR
# define KindNoPointers GO_NO_POINTERS
// PtrType aka __go_ptr_type
# define elem __element_type
2010-12-03 05:34:57 +01:00
2011-11-28 06:45:49 +01:00
# ifdef USING_SPLIT_STACK
extern void * __splitstack_find ( void * , void * , size_t * , void * * , void * * ,
void * * ) ;
extern void * __splitstack_find_context ( void * context [ 10 ] , size_t * , void * * ,
void * * , void * * ) ;
# endif
2010-12-03 05:34:57 +01:00
enum {
2011-03-17 00:05:44 +01:00
Debug = 0 ,
2011-10-27 01:57:58 +02:00
DebugMark = 0 , // run second pass to check mark
2013-07-16 08:54:42 +02:00
CollectStats = 0 ,
2014-06-05 01:15:33 +02:00
ScanStackByFrames = 1 ,
2013-07-16 08:54:42 +02:00
IgnorePreciseGC = 0 ,
2011-10-27 01:57:58 +02:00
2011-03-17 00:05:44 +01:00
// Four bits per word (see #defines below).
wordsPerBitmapWord = sizeof ( void * ) * 8 / 4 ,
bitShift = sizeof ( void * ) * 8 / 4 ,
2012-12-22 02:15:33 +01:00
handoffThreshold = 4 ,
IntermediateBufferCapacity = 64 ,
2013-01-29 21:52:43 +01:00
// Bits in type information
PRECISE = 1 ,
LOOP = 2 ,
PC_BITS = PRECISE | LOOP ,
2013-11-06 20:49:01 +01:00
// Pointer map
BitsPerPointer = 2 ,
BitsNoPointer = 0 ,
BitsPointer = 1 ,
BitsIface = 2 ,
BitsEface = 3 ,
2010-12-03 05:34:57 +01:00
} ;
2014-06-05 01:15:33 +02:00
static struct
{
Lock ;
void * head ;
} pools ;
void sync_runtime_registerPool ( void * * )
__asm__ ( GOSYM_PREFIX " sync.runtime_registerPool " ) ;
void
sync_runtime_registerPool ( void * * p )
{
runtime_lock ( & pools ) ;
p [ 0 ] = pools . head ;
pools . head = p ;
runtime_unlock ( & pools ) ;
}
static void
clearpools ( void )
{
void * * p , * * next ;
for ( p = pools . head ; p ! = nil ; p = next ) {
next = p [ 0 ] ;
p [ 0 ] = nil ; // next
p [ 1 ] = nil ; // slice
p [ 2 ] = nil ;
p [ 3 ] = nil ;
}
pools . head = nil ;
}
2011-03-17 00:05:44 +01:00
// Bits in per-word bitmap.
// #defines because enum might not be able to hold the values.
//
// Each word in the bitmap describes wordsPerBitmapWord words
// of heap memory. There are 4 bitmap bits dedicated to each heap word,
// so on a 64-bit system there is one bitmap word per 16 heap words.
// The bits in the word are packed together by type first, then by
// heap location, so each 64-bit bitmap word consists of, from top to bottom,
// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
2014-06-05 01:15:33 +02:00
// then the 16 bitScan/bitBlockBoundary bits, then the 16 bitAllocated bits.
2011-03-17 00:05:44 +01:00
// This layout makes it easier to iterate over the bits of a given type.
//
// The bitmap starts at mheap.arena_start and extends *backward* from
// there. On a 64-bit system the off'th word in the arena is tracked by
// the off/16+1'th word before mheap.arena_start. (On a 32-bit system,
// the only difference is that the divisor is 8.)
//
// To pull out the bits corresponding to a given pointer p, we use:
//
// off = p - (uintptr*)mheap.arena_start; // word offset
// b = (uintptr*)mheap.arena_start - off/wordsPerBitmapWord - 1;
// shift = off % wordsPerBitmapWord
// bits = *b >> shift;
// /* then test bits & bitAllocated, bits & bitMarked, etc. */
//
2014-06-05 01:15:33 +02:00
# define bitAllocated ((uintptr)1<<(bitShift*0)) /* block start; eligible for garbage collection */
# define bitScan ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
2011-03-17 00:05:44 +01:00
# define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
# define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
2014-06-05 01:15:33 +02:00
# define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set - mark for FlagNoGC objects */
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
# define bitMask (bitAllocated | bitScan | bitMarked | bitSpecial)
2011-03-17 00:05:44 +01:00
2012-03-02 21:01:37 +01:00
// Holding worldsema grants an M the right to try to stop the world.
// The procedure is:
//
// runtime_semacquire(&runtime_worldsema);
// m->gcing = 1;
// runtime_stoptheworld();
//
// ... do stuff ...
//
// m->gcing = 0;
// runtime_semrelease(&runtime_worldsema);
// runtime_starttheworld();
//
uint32 runtime_worldsema = 1 ;
2012-12-22 02:15:33 +01:00
// The size of Workbuf is N*PageSize.
2011-03-17 00:05:44 +01:00
typedef struct Workbuf Workbuf ;
struct Workbuf
2010-12-03 05:34:57 +01:00
{
2012-12-22 02:15:33 +01:00
# define SIZE (2*PageSize-sizeof(LFNode)-sizeof(uintptr))
LFNode node ; // must be first
2011-10-27 01:57:58 +02:00
uintptr nobj ;
2012-12-22 02:15:33 +01:00
Obj obj [ SIZE / sizeof ( Obj ) - 1 ] ;
uint8 _padding [ SIZE % sizeof ( Obj ) + sizeof ( Obj ) ] ;
# undef SIZE
2011-10-27 01:57:58 +02:00
} ;
typedef struct Finalizer Finalizer ;
struct Finalizer
{
2013-06-19 01:49:49 +02:00
FuncVal * fn ;
2011-10-27 01:57:58 +02:00
void * arg ;
const struct __go_func_type * ft ;
2013-11-06 20:49:01 +01:00
const struct __go_ptr_type * ot ;
2011-10-27 01:57:58 +02:00
} ;
typedef struct FinBlock FinBlock ;
struct FinBlock
{
FinBlock * alllink ;
FinBlock * next ;
int32 cnt ;
int32 cap ;
Finalizer fin [ 1 ] ;
2010-12-03 05:34:57 +01:00
} ;
2011-11-28 06:45:49 +01:00
static G * fing ;
2011-10-27 01:57:58 +02:00
static FinBlock * finq ; // list of finalizers that are to be executed
static FinBlock * finc ; // cache of free blocks
static FinBlock * allfin ; // list of all blocks
static Lock finlock ;
2010-12-03 05:34:57 +01:00
static int32 fingwait ;
static void runfinq ( void * ) ;
2011-03-17 00:05:44 +01:00
static Workbuf * getempty ( Workbuf * ) ;
static Workbuf * getfull ( Workbuf * ) ;
2011-10-27 01:57:58 +02:00
static void putempty ( Workbuf * ) ;
static Workbuf * handoff ( Workbuf * ) ;
2013-07-16 08:54:42 +02:00
static void gchelperstart ( void ) ;
2011-10-27 01:57:58 +02:00
static struct {
2012-10-23 06:31:11 +02:00
uint64 full ; // lock-free list of full blocks
uint64 empty ; // lock-free list of empty blocks
byte pad0 [ CacheLineSize ] ; // prevents false-sharing between full/empty and nproc/nwait
2011-10-27 01:57:58 +02:00
uint32 nproc ;
volatile uint32 nwait ;
volatile uint32 ndone ;
2012-10-23 06:31:11 +02:00
volatile uint32 debugmarkdone ;
2011-10-27 01:57:58 +02:00
Note alldone ;
2012-10-23 06:31:11 +02:00
ParFor * markfor ;
ParFor * sweepfor ;
2011-10-27 01:57:58 +02:00
Lock ;
byte * chunk ;
uintptr nchunk ;
2012-10-23 06:31:11 +02:00
2012-12-22 02:15:33 +01:00
Obj * roots ;
2012-10-23 06:31:11 +02:00
uint32 nroot ;
uint32 rootcap ;
2014-01-17 23:43:03 +01:00
} work __attribute__ ( ( aligned ( 8 ) ) ) ;
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR ,
2013-07-16 08:54:42 +02:00
GC_CHAN ,
2014-06-05 01:15:33 +02:00
GC_G_PTR ,
2013-07-16 08:54:42 +02:00
GC_NUM_INSTR2
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static struct {
struct {
uint64 sum ;
uint64 cnt ;
} ptr ;
uint64 nbytes ;
struct {
uint64 sum ;
uint64 cnt ;
uint64 notype ;
uint64 typelookup ;
} obj ;
uint64 rescan ;
uint64 rescanbytes ;
uint64 instr [ GC_NUM_INSTR2 ] ;
uint64 putempty ;
uint64 getfull ;
2013-11-06 20:49:01 +01:00
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} flushptrbuf ;
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} markonly ;
2013-07-16 08:54:42 +02:00
} gcstats ;
// markonly marks an object. It returns true if the object
// has been marked by this function, false otherwise.
// This function doesn't append the object to any buffer.
static bool
markonly ( void * obj )
{
byte * p ;
2013-11-06 20:49:01 +01:00
uintptr * bitp , bits , shift , x , xbits , off , j ;
2013-07-16 08:54:42 +02:00
MSpan * s ;
PageID k ;
// Words outside the arena cannot be pointers.
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj < runtime_mheap . arena_start | | ( byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return false ;
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
obj = ( void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
// Find bits for this word.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
// Pointing at the beginning of a block?
2013-11-06 20:49:01 +01:00
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundbit , 1 ) ;
2013-07-16 08:54:42 +02:00
goto found ;
2013-11-06 20:49:01 +01:00
}
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
shift = j ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundword , 1 ) ;
goto found ;
}
}
2013-07-16 08:54:42 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) runtime_mheap . arena_start > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
if ( s = = nil | | k < s - > start | | ( byte * ) obj > = s - > limit | | s - > state ! = MSpanInUse )
2013-07-16 08:54:42 +02:00
return false ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
uintptr size = s - > elemsize ;
int32 i = ( ( byte * ) obj - p ) / size ;
obj = p + i * size ;
}
// Now that we know the object header, reload bits.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundspan , 1 ) ;
2013-07-16 08:54:42 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
return false ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
return false ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
}
}
// The object is now marked
return true ;
}
// PtrTarget is a structure used by intermediate buffers.
2012-12-22 02:15:33 +01:00
// The intermediate buffers hold GC data before it
// is moved/flushed to the work buffer (Workbuf).
// The size of an intermediate buffer is very small,
// such as 32 or 64 elements.
2013-01-29 21:52:43 +01:00
typedef struct PtrTarget PtrTarget ;
2012-12-22 02:15:33 +01:00
struct PtrTarget
{
void * p ;
uintptr ti ;
} ;
2014-06-05 01:15:33 +02:00
typedef struct Scanbuf Scanbuf ;
struct Scanbuf
{
struct {
PtrTarget * begin ;
PtrTarget * end ;
PtrTarget * pos ;
} ptr ;
struct {
Obj * begin ;
Obj * end ;
Obj * pos ;
} obj ;
Workbuf * wbuf ;
Obj * wp ;
uintptr nobj ;
} ;
2013-01-29 21:52:43 +01:00
typedef struct BufferList BufferList ;
2012-12-22 02:15:33 +01:00
struct BufferList
{
2013-01-29 21:52:43 +01:00
PtrTarget ptrtarget [ IntermediateBufferCapacity ] ;
2013-07-16 08:54:42 +02:00
Obj obj [ IntermediateBufferCapacity ] ;
uint32 busy ;
byte pad [ CacheLineSize ] ;
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static BufferList bufferList [ MaxGcproc ] ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
static Type * itabtype ;
static void enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj ) ;
2012-12-22 02:15:33 +01:00
// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
// while the work buffer contains blocks which have been marked
// and are prepared to be scanned by the garbage collector.
//
// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
//
// A simplified drawing explaining how the todo-list moves from a structure to another:
//
// scanblock
// (find pointers)
// Obj ------> PtrTarget (pointer targets)
// ↑ |
2013-07-16 08:54:42 +02:00
// | |
// `----------'
// flushptrbuf
// (find block start, mark and enqueue)
2010-12-03 05:34:57 +01:00
static void
2014-06-05 01:15:33 +02:00
flushptrbuf ( Scanbuf * sbuf )
2010-12-03 05:34:57 +01:00
{
2012-12-22 02:15:33 +01:00
byte * p , * arena_start , * obj ;
2013-01-29 21:52:43 +01:00
uintptr size , * bitp , bits , shift , j , x , xbits , off , nobj , ti , n ;
2011-03-17 00:05:44 +01:00
MSpan * s ;
PageID k ;
2012-12-22 02:15:33 +01:00
Obj * wp ;
2011-03-17 00:05:44 +01:00
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
PtrTarget * ptrbuf ;
2013-01-29 21:52:43 +01:00
PtrTarget * ptrbuf_end ;
2011-05-20 02:18:15 +02:00
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
ptrbuf = sbuf - > ptr . begin ;
ptrbuf_end = sbuf - > ptr . pos ;
n = ptrbuf_end - sbuf - > ptr . begin ;
sbuf - > ptr . pos = sbuf - > ptr . begin ;
2011-03-17 00:05:44 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . ptr . sum , n ) ;
runtime_xadd64 ( & gcstats . ptr . cnt , 1 ) ;
}
2012-12-22 02:15:33 +01:00
// If buffer is nearly full, get a new one.
if ( wbuf = = nil | | nobj + n > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
if ( n > = nelem ( wbuf - > obj ) )
runtime_throw ( " ptrbuf has to be smaller than WorkBuf " ) ;
2011-03-17 00:05:44 +01:00
}
2010-12-03 05:34:57 +01:00
2014-06-05 01:15:33 +02:00
while ( ptrbuf < ptrbuf_end ) {
obj = ptrbuf - > p ;
ti = ptrbuf - > ti ;
ptrbuf + + ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if ( Debug > 1 ) {
if ( obj < runtime_mheap . arena_start | | obj > = runtime_mheap . arena_used )
runtime_throw ( " object is outside of mheap " ) ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
if ( ( ( uintptr ) obj & ( ( uintptr ) PtrSize - 1 ) ) ! = 0 ) {
obj = ( void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
ti = 0 ;
}
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Find bits for this word.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Pointing at the beginning of a block?
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundbit , 1 ) ;
goto found ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
ti = 0 ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
obj = ( byte * ) obj - ( shift - j ) * PtrSize ;
shift = j ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . flushptrbuf . foundword , 1 ) ;
2011-03-17 00:05:44 +01:00
goto found ;
2013-11-06 20:49:01 +01:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
if ( s = = nil | | k < s - > start | | obj > = s - > limit | | s - > state ! = MSpanInUse )
continue ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
size = s - > elemsize ;
int32 i = ( ( byte * ) obj - p ) / size ;
obj = p + i * size ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Now that we know the object header, reload bits.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundspan , 1 ) ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
continue ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
goto continue_obj ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
2013-07-16 08:54:42 +02:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// If object has no pointers, don't need to scan further.
if ( ( bits & bitScan ) = = 0 )
continue ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = ( uintptr ) obj > > PageShift ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
PREFETCH ( obj ) ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
* wp = ( Obj ) { obj , s - > elemsize , ti } ;
wp + + ;
nobj + + ;
continue_obj : ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
2012-12-22 02:15:33 +01:00
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2012-12-22 02:15:33 +01:00
}
2013-07-16 08:54:42 +02:00
static void
2014-06-05 01:15:33 +02:00
flushobjbuf ( Scanbuf * sbuf )
2013-07-16 08:54:42 +02:00
{
uintptr nobj , off ;
Obj * wp , obj ;
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
Obj * objbuf ;
2013-07-16 08:54:42 +02:00
Obj * objbuf_end ;
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2013-07-16 08:54:42 +02:00
2014-06-05 01:15:33 +02:00
objbuf = sbuf - > obj . begin ;
objbuf_end = sbuf - > obj . pos ;
sbuf - > obj . pos = sbuf - > obj . begin ;
2013-07-16 08:54:42 +02:00
while ( objbuf < objbuf_end ) {
obj = * objbuf + + ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
continue ;
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
}
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2013-07-16 08:54:42 +02:00
}
2012-12-22 02:15:33 +01:00
// Program that scans the whole block and treats every block element as a potential pointer
static uintptr defaultProg [ 2 ] = { PtrSize , GC_DEFAULT_PTR } ;
2013-07-16 08:54:42 +02:00
#if 0
// Hchan program
static uintptr chanProg [ 2 ] = { 0 , GC_CHAN } ;
# endif
2014-06-05 01:15:33 +02:00
#if 0
// G* program
static uintptr gptrProg [ 2 ] = { 0 , GC_G_PTR } ;
# endif
2013-01-29 21:52:43 +01:00
// Local variables of a program fragment or loop
typedef struct Frame Frame ;
struct Frame {
uintptr count , elemsize , b ;
uintptr * loop_or_ret ;
} ;
2013-07-16 08:54:42 +02:00
// Sanity check for the derived type info objti.
static void
checkptr ( void * obj , uintptr objti )
{
uintptr type , tisize , i , x ;
byte * objstart ;
Type * t ;
MSpan * s ;
if ( ! Debug )
runtime_throw ( " checkptr is debug only " ) ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj < runtime_mheap . arena_start | | ( byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return ;
type = runtime_gettype ( obj ) ;
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
if ( t = = nil )
return ;
x = ( uintptr ) obj > > PageShift ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) ( runtime_mheap . arena_start ) > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
2013-07-16 08:54:42 +02:00
objstart = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass ! = 0 ) {
i = ( ( byte * ) obj - objstart ) / s - > elemsize ;
objstart + = i * s - > elemsize ;
}
tisize = * ( uintptr * ) objti ;
// Sanity check for object size: it should fit into the memory block.
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj + tisize > objstart + s - > elemsize ) {
runtime_printf ( " object of type '%S' at %p/%p does not fit in block %p/%p \n " ,
* t - > string , obj , tisize , objstart , s - > elemsize ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
2013-11-06 20:49:01 +01:00
}
2013-07-16 08:54:42 +02:00
if ( obj ! = objstart )
return ;
// If obj points to the beginning of the memory block,
// check type info as well.
if ( t - > string = = nil | |
// Gob allocates unsafe pointers for indirection.
( runtime_strcmp ( ( const char * ) t - > string - > str , ( const char * ) " unsafe.Pointer " ) & &
// Runtime and gc think differently about closures.
runtime_strstr ( ( const char * ) t - > string - > str , ( const char * ) " struct { F uintptr " ) ! = ( const char * ) t - > string - > str ) ) {
#if 0
pc1 = ( uintptr * ) objti ;
pc2 = ( uintptr * ) t - > gc ;
// A simple best-effort check until first GC_END.
for ( j = 1 ; pc1 [ j ] ! = GC_END & & pc2 [ j ] ! = GC_END ; j + + ) {
if ( pc1 [ j ] ! = pc2 [ j ] ) {
runtime_printf ( " invalid gc type info for '%s' at %p, type info %p, block info %p \n " ,
2013-11-06 20:49:01 +01:00
t - > string ? ( const int8 * ) t - > string - > str : ( const int8 * ) " ? " , j , pc1 [ j ] , pc2 [ j ] ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
}
}
# endif
}
}
2012-12-22 02:15:33 +01:00
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
//
// wbuf: current work buffer
// wp: storage for next queued pointer (write pointer)
// nobj: number of queued objects
static void
scanblock ( Workbuf * wbuf , Obj * wp , uintptr nobj , bool keepworking )
{
byte * b , * arena_start , * arena_used ;
2013-07-16 08:54:42 +02:00
uintptr n , i , end_b , elemsize , size , ti , objti , count /* , type */ ;
2013-01-29 21:52:43 +01:00
uintptr * pc , precise_type , nominal_size ;
2013-07-16 08:54:42 +02:00
#if 0
2013-11-06 20:49:01 +01:00
uintptr * chan_ret , chancap ;
2013-07-16 08:54:42 +02:00
# endif
2012-12-22 02:15:33 +01:00
void * obj ;
2013-01-29 21:52:43 +01:00
const Type * t ;
Slice * sliceptr ;
Frame * stack_ptr , stack_top , stack [ GC_STACK_CAPACITY + 4 ] ;
BufferList * scanbuffers ;
2014-06-05 01:15:33 +02:00
Scanbuf sbuf ;
2013-01-29 21:52:43 +01:00
Eface * eface ;
Iface * iface ;
2013-07-16 08:54:42 +02:00
#if 0
Hchan * chan ;
ChanType * chantype ;
# endif
2012-12-22 02:15:33 +01:00
if ( sizeof ( Workbuf ) % PageSize ! = 0 )
runtime_throw ( " scanblock: size of Workbuf is suboptimal " ) ;
// Memory arena parameters.
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
arena_used = runtime_mheap . arena_used ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
stack_ptr = stack + nelem ( stack ) - 1 ;
2014-06-05 01:15:33 +02:00
2013-01-29 21:52:43 +01:00
precise_type = false ;
nominal_size = 0 ;
2014-06-05 01:15:33 +02:00
// Initialize sbuf
scanbuffers = & bufferList [ runtime_m ( ) - > helpgc ] ;
sbuf . ptr . begin = sbuf . ptr . pos = & scanbuffers - > ptrtarget [ 0 ] ;
sbuf . ptr . end = sbuf . ptr . begin + nelem ( scanbuffers - > ptrtarget ) ;
sbuf . obj . begin = sbuf . obj . pos = & scanbuffers - > obj [ 0 ] ;
sbuf . obj . end = sbuf . obj . begin + nelem ( scanbuffers - > obj ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
sbuf . wbuf = wbuf ;
sbuf . wp = wp ;
sbuf . nobj = nobj ;
2013-07-16 08:54:42 +02:00
// (Silence the compiler)
#if 0
chan = nil ;
chantype = nil ;
chan_ret = nil ;
# endif
2012-12-22 02:15:33 +01:00
goto next_block ;
for ( ; ; ) {
// Each iteration scans the block b of length n, queueing pointers in
// the work buffer.
if ( Debug > 1 ) {
runtime_printf ( " scanblock %p %D \n " , b , ( int64 ) n ) ;
}
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . nbytes , n ) ;
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . obj . sum , sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
runtime_xadd64 ( & gcstats . obj . cnt , 1 ) ;
}
if ( ti ! = 0 & & false ) {
2013-01-29 21:52:43 +01:00
pc = ( uintptr * ) ( ti & ~ ( uintptr ) PC_BITS ) ;
precise_type = ( ti & PRECISE ) ;
stack_top . elemsize = pc [ 0 ] ;
if ( ! precise_type )
nominal_size = pc [ 0 ] ;
if ( ti & LOOP ) {
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . loop_or_ret = pc + 1 ;
} else {
stack_top . count = 1 ;
}
2013-07-16 08:54:42 +02:00
if ( Debug ) {
// Simple sanity check for provided type info ti:
// The declared size of the object must be not larger than the actual size
// (it can be smaller due to inferior pointers).
// It's difficult to make a comprehensive check due to inferior pointers,
// reflection, gob, etc.
if ( pc [ 0 ] > n ) {
runtime_printf ( " invalid gc type info: type info size %p, block size %p \n " , pc [ 0 ] , n ) ;
runtime_throw ( " invalid gc type info " ) ;
}
}
} else if ( UseSpanType & & false ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . notype , 1 ) ;
2013-01-29 21:52:43 +01:00
#if 0
type = runtime_gettype ( b ) ;
if ( type ! = 0 ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . typelookup , 1 ) ;
2013-01-29 21:52:43 +01:00
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
switch ( type & ( PtrSize - 1 ) ) {
case TypeInfo_SingleObject :
pc = ( uintptr * ) t - > gc ;
precise_type = true ; // type information about 'b' is precise
stack_top . count = 1 ;
stack_top . elemsize = pc [ 0 ] ;
break ;
case TypeInfo_Array :
pc = ( uintptr * ) t - > gc ;
if ( pc [ 0 ] = = 0 )
goto next_block ;
precise_type = true ; // type information about 'b' is precise
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . elemsize = pc [ 0 ] ;
stack_top . loop_or_ret = pc + 1 ;
break ;
2013-07-16 08:54:42 +02:00
case TypeInfo_Chan :
chan = ( Hchan * ) b ;
chantype = ( ChanType * ) t ;
chan_ret = nil ;
pc = chanProg ;
2013-01-29 21:52:43 +01:00
break ;
default :
runtime_throw ( " scanblock: invalid type " ) ;
return ;
}
} else {
pc = defaultProg ;
}
# endif
} else {
pc = defaultProg ;
}
2012-12-22 02:15:33 +01:00
2013-07-16 08:54:42 +02:00
if ( IgnorePreciseGC )
pc = defaultProg ;
2012-12-22 02:15:33 +01:00
pc + + ;
stack_top . b = ( uintptr ) b ;
end_b = ( uintptr ) b + n - PtrSize ;
2013-01-29 21:52:43 +01:00
for ( ; ; ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . instr [ pc [ 0 ] ] , 1 ) ;
2013-01-29 21:52:43 +01:00
obj = nil ;
objti = 0 ;
2012-12-22 02:15:33 +01:00
switch ( pc [ 0 ] ) {
2013-01-29 21:52:43 +01:00
case GC_PTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
objti = pc [ 2 ] ;
pc + = 3 ;
2013-07-16 08:54:42 +02:00
if ( Debug )
checkptr ( obj , objti ) ;
2013-01-29 21:52:43 +01:00
break ;
case GC_SLICE :
sliceptr = ( Slice * ) ( stack_top . b + pc [ 1 ] ) ;
if ( sliceptr - > cap ! = 0 ) {
obj = sliceptr - > array ;
2013-07-16 08:54:42 +02:00
// Can't use slice element type for scanning,
// because if it points to an array embedded
// in the beginning of a struct,
// we will scan the whole struct as the slice.
// So just obtain type info from heap.
2013-01-29 21:52:43 +01:00
}
pc + = 3 ;
break ;
case GC_APTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
break ;
case GC_STRING :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
2013-07-16 08:54:42 +02:00
markonly ( obj ) ;
2013-01-29 21:52:43 +01:00
pc + = 2 ;
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
case GC_EFACE :
eface = ( Eface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
2013-07-16 08:54:42 +02:00
if ( eface - > type = = nil )
continue ;
// eface->type
t = eface - > type ;
if ( ( const byte * ) t > = arena_start & & ( const byte * ) t < arena_used ) {
union { const Type * tc ; Type * tr ; } u ;
u . tc = t ;
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { u . tr , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
// eface->__object
if ( ( byte * ) eface - > __object > = arena_start & & ( byte * ) eface - > __object < arena_used ) {
2013-01-29 21:52:43 +01:00
if ( t - > __size < = sizeof ( void * ) ) {
if ( ( t - > kind & KindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = eface - > __object ;
if ( ( t - > kind & ~ KindNoPointers ) = = KindPtr )
// objti = (uintptr)((PtrType*)t)->elem->gc;
objti = 0 ;
} else {
obj = eface - > __object ;
// objti = (uintptr)t->gc;
objti = 0 ;
}
}
break ;
case GC_IFACE :
iface = ( Iface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
if ( iface - > tab = = nil )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
// iface->tab
if ( ( byte * ) iface - > tab > = arena_start & & ( byte * ) iface - > tab < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { iface - > tab , /* (uintptr)itabtype->gc */ 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
// iface->data
if ( ( byte * ) iface - > __object > = arena_start & & ( byte * ) iface - > __object < arena_used ) {
// t = iface->tab->type;
t = nil ;
if ( t - > __size < = sizeof ( void * ) ) {
if ( ( t - > kind & KindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = iface - > __object ;
if ( ( t - > kind & ~ KindNoPointers ) = = KindPtr )
// objti = (uintptr)((const PtrType*)t)->elem->gc;
objti = 0 ;
} else {
obj = iface - > __object ;
// objti = (uintptr)t->gc;
objti = 0 ;
}
}
break ;
2012-12-22 02:15:33 +01:00
case GC_DEFAULT_PTR :
2013-07-16 08:54:42 +02:00
while ( stack_top . b < = end_b ) {
obj = * ( byte * * ) stack_top . b ;
2012-12-22 02:15:33 +01:00
stack_top . b + = PtrSize ;
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
}
}
2013-01-29 21:52:43 +01:00
goto next_block ;
case GC_END :
if ( - - stack_top . count ! = 0 ) {
// Next iteration of a loop if possible.
2013-07-16 08:54:42 +02:00
stack_top . b + = stack_top . elemsize ;
if ( stack_top . b + stack_top . elemsize < = end_b + PtrSize ) {
2013-01-29 21:52:43 +01:00
pc = stack_top . loop_or_ret ;
continue ;
}
i = stack_top . b ;
} else {
// Stack pop if possible.
if ( stack_ptr + 1 < stack + nelem ( stack ) ) {
pc = stack_top . loop_or_ret ;
stack_top = * ( + + stack_ptr ) ;
continue ;
}
i = ( uintptr ) b + nominal_size ;
}
if ( ! precise_type ) {
// Quickly scan [b+i,b+n) for possible pointers.
for ( ; i < = end_b ; i + = PtrSize ) {
if ( * ( byte * * ) i ! = nil ) {
// Found a value that may be a pointer.
// Do a rescan of the entire block.
2014-06-05 01:15:33 +02:00
enqueue ( ( Obj ) { b , n , 0 } , & sbuf . wbuf , & sbuf . wp , & sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . rescan , 1 ) ;
runtime_xadd64 ( & gcstats . rescanbytes , n ) ;
}
2013-01-29 21:52:43 +01:00
break ;
}
}
}
goto next_block ;
case GC_ARRAY_START :
i = stack_top . b + pc [ 1 ] ;
count = pc [ 2 ] ;
elemsize = pc [ 3 ] ;
pc + = 4 ;
// Stack push.
* stack_ptr - - = stack_top ;
stack_top = ( Frame ) { count , elemsize , i , pc } ;
continue ;
case GC_ARRAY_NEXT :
if ( - - stack_top . count ! = 0 ) {
stack_top . b + = stack_top . elemsize ;
pc = stack_top . loop_or_ret ;
} else {
// Stack pop.
stack_top = * ( + + stack_ptr ) ;
pc + = 1 ;
}
continue ;
case GC_CALL :
// Stack push.
* stack_ptr - - = stack_top ;
stack_top = ( Frame ) { 1 , 0 , stack_top . b + pc [ 1 ] , pc + 3 /*return address*/ } ;
2013-07-16 08:54:42 +02:00
pc = ( uintptr * ) ( ( byte * ) pc + * ( int32 * ) ( pc + 2 ) ) ; // target of the CALL instruction
2013-01-29 21:52:43 +01:00
continue ;
case GC_REGION :
obj = ( void * ) ( stack_top . b + pc [ 1 ] ) ;
2013-07-16 08:54:42 +02:00
size = pc [ 2 ] ;
objti = pc [ 3 ] ;
2013-01-29 21:52:43 +01:00
pc + = 4 ;
2013-07-16 08:54:42 +02:00
2014-06-05 01:15:33 +02:00
* sbuf . obj . pos + + = ( Obj ) { obj , size , objti } ;
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
continue ;
#if 0
case GC_CHAN_PTR :
chan = * ( Hchan * * ) ( stack_top . b + pc [ 1 ] ) ;
if ( chan = = nil ) {
pc + = 3 ;
continue ;
}
if ( markonly ( chan ) ) {
chantype = ( ChanType * ) pc [ 2 ] ;
if ( ! ( chantype - > elem - > kind & KindNoPointers ) ) {
// Start chanProg.
chan_ret = pc + 3 ;
pc = chanProg + 1 ;
continue ;
}
}
pc + = 3 ;
continue ;
case GC_CHAN :
// There are no heap pointers in struct Hchan,
// so we can ignore the leading sizeof(Hchan) bytes.
if ( ! ( chantype - > elem - > kind & KindNoPointers ) ) {
// Channel's buffer follows Hchan immediately in memory.
// Size of buffer (cap(c)) is second int in the chan struct.
chancap = ( ( uintgo * ) chan ) [ 1 ] ;
if ( chancap > 0 ) {
// TODO(atom): split into two chunks so that only the
// in-use part of the circular buffer is scanned.
// (Channel routines zero the unused part, so the current
// code does not lead to leaks, it's just a little inefficient.)
2014-06-05 01:15:33 +02:00
* sbuf . obj . pos + + = ( Obj ) { ( byte * ) chan + runtime_Hchansize , chancap * chantype - > elem - > size ,
2013-07-16 08:54:42 +02:00
( uintptr ) chantype - > elem - > gc | PRECISE | LOOP } ;
2014-06-05 01:15:33 +02:00
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
}
if ( chan_ret = = nil )
goto next_block ;
pc = chan_ret ;
continue ;
# endif
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
#if 0
case GC_G_PTR :
obj = ( void * ) stack_top . b ;
scanstack ( obj , & sbuf ) ;
goto next_block ;
# endif
2012-12-22 02:15:33 +01:00
default :
runtime_throw ( " scanblock: invalid GC instruction " ) ;
return ;
}
2013-01-29 21:52:43 +01:00
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , objti } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
}
2012-12-22 02:15:33 +01:00
next_block :
2011-03-17 00:05:44 +01:00
// Done scanning [b, b+n). Prepare for the next iteration of
2013-01-29 21:52:43 +01:00
// the loop by setting b, n, ti to the parameters for the next block.
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
flushptrbuf ( & sbuf ) ;
flushobjbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
2012-12-22 02:15:33 +01:00
if ( ! keepworking ) {
2014-06-05 01:15:33 +02:00
if ( sbuf . wbuf )
putempty ( sbuf . wbuf ) ;
return ;
2012-12-22 02:15:33 +01:00
}
// Emptied our buffer: refill.
2014-06-05 01:15:33 +02:00
sbuf . wbuf = getfull ( sbuf . wbuf ) ;
if ( sbuf . wbuf = = nil )
return ;
sbuf . nobj = sbuf . wbuf - > nobj ;
sbuf . wp = sbuf . wbuf - > obj + sbuf . wbuf - > nobj ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
2012-12-22 02:15:33 +01:00
// Fetch b from the work buffer.
2014-06-05 01:15:33 +02:00
- - sbuf . wp ;
b = sbuf . wp - > p ;
n = sbuf . wp - > n ;
ti = sbuf . wp - > ti ;
sbuf . nobj - - ;
2011-03-17 00:05:44 +01:00
}
}
2011-10-27 01:57:58 +02:00
// debug_scanblock is the debug copy of scanblock.
// it is simpler, slower, single-threaded, recursive,
// and uses bitSpecial as the mark bit.
static void
2012-10-23 06:31:11 +02:00
debug_scanblock ( byte * b , uintptr n )
2011-10-27 01:57:58 +02:00
{
byte * obj , * p ;
void * * vp ;
uintptr size , * bitp , bits , shift , i , xbits , off ;
MSpan * s ;
if ( ! DebugMark )
runtime_throw ( " debug_scanblock without DebugMark " ) ;
2012-10-23 06:31:11 +02:00
if ( ( intptr ) n < 0 ) {
runtime_printf ( " debug_scanblock %p %D \n " , b , ( int64 ) n ) ;
2011-10-27 01:57:58 +02:00
runtime_throw ( " debug_scanblock " ) ;
}
// Align b to a word boundary.
off = ( uintptr ) b & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
b + = PtrSize - off ;
n - = PtrSize - off ;
}
vp = ( void * * ) b ;
n / = PtrSize ;
for ( i = 0 ; i < ( uintptr ) n ; i + + ) {
obj = ( byte * ) vp [ i ] ;
// Words outside the arena cannot be pointers.
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj < runtime_mheap . arena_start | | ( byte * ) obj > = runtime_mheap . arena_used )
2011-10-27 01:57:58 +02:00
continue ;
// Round down to word boundary.
obj = ( void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
// Consult span table to find beginning.
2013-11-06 20:49:01 +01:00
s = runtime_MHeap_LookupMaybe ( & runtime_mheap , obj ) ;
2011-10-27 01:57:58 +02:00
if ( s = = nil )
continue ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
2012-12-22 02:15:33 +01:00
size = s - > elemsize ;
2011-10-27 01:57:58 +02:00
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
int32 i = ( ( byte * ) obj - p ) / size ;
obj = p + i * size ;
}
// Now that we know the object header, reload bits.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-10-27 01:57:58 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// If not allocated or already marked, done.
if ( ( bits & bitAllocated ) = = 0 | | ( bits & bitSpecial ) ! = 0 ) // NOTE: bitSpecial not bitMarked
continue ;
* bitp | = bitSpecial < < shift ;
if ( ! ( bits & bitMarked ) )
runtime_printf ( " found unmarked block %p in %p \n " , obj , vp + i ) ;
// If object has no pointers, don't need to scan further.
2014-06-05 01:15:33 +02:00
if ( ( bits & bitScan ) = = 0 )
2011-10-27 01:57:58 +02:00
continue ;
debug_scanblock ( obj , size ) ;
}
}
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
// Append obj to the work buffer.
// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
static void
enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj )
{
uintptr nobj , off ;
Obj * wp ;
Workbuf * wbuf ;
if ( Debug > 1 )
runtime_printf ( " append obj(%p %D %p) \n " , obj . p , ( int64 ) obj . n , obj . ti ) ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
return ;
// Load work buffer state
wp = * _wp ;
wbuf = * _wbuf ;
nobj = * _nobj ;
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
// Save work buffer state
* _wp = wp ;
* _wbuf = wbuf ;
* _nobj = nobj ;
}
2012-10-23 06:31:11 +02:00
static void
markroot ( ParFor * desc , uint32 i )
{
2012-12-22 02:15:33 +01:00
Obj * wp ;
Workbuf * wbuf ;
uintptr nobj ;
2012-10-23 06:31:11 +02:00
USED ( & desc ) ;
2012-12-22 02:15:33 +01:00
wp = nil ;
wbuf = nil ;
nobj = 0 ;
enqueue ( work . roots [ i ] , & wbuf , & wp , & nobj ) ;
scanblock ( wbuf , wp , nobj , false ) ;
2012-10-23 06:31:11 +02:00
}
2011-03-17 00:05:44 +01:00
// Get an empty work buffer off the work.empty list,
// allocating new buffers as needed.
static Workbuf *
getempty ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
runtime_lfstackpush ( & work . full , & b - > node ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . empty ) ;
if ( b = = nil ) {
// Need to allocate.
runtime_lock ( & work ) ;
if ( work . nchunk < sizeof * b ) {
work . nchunk = 1 < < 20 ;
2013-11-06 20:49:01 +01:00
work . chunk = runtime_SysAlloc ( work . nchunk , & mstats . gc_sys ) ;
2013-07-16 08:54:42 +02:00
if ( work . chunk = = nil )
runtime_throw ( " runtime: cannot allocate memory " ) ;
2011-10-27 01:57:58 +02:00
}
2012-10-23 06:31:11 +02:00
b = ( Workbuf * ) work . chunk ;
work . chunk + = sizeof * b ;
work . nchunk - = sizeof * b ;
runtime_unlock ( & work ) ;
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
b - > nobj = 0 ;
2011-03-17 00:05:44 +01:00
return b ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
static void
putempty ( Workbuf * b )
{
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . putempty , 1 ) ;
2012-10-23 06:31:11 +02:00
runtime_lfstackpush ( & work . empty , & b - > node ) ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
// Get a full work buffer off the work.full list, or return nil.
static Workbuf *
getfull ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 i ;
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . getfull , 1 ) ;
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
runtime_lfstackpush ( & work . empty , & b - > node ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil | | work . nproc = = 1 )
2011-10-27 01:57:58 +02:00
return b ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
runtime_xadd ( & work . nwait , + 1 ) ;
for ( i = 0 ; ; i + + ) {
2012-10-23 06:31:11 +02:00
if ( work . full ! = 0 ) {
runtime_xadd ( & work . nwait , - 1 ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil )
return b ;
runtime_xadd ( & work . nwait , + 1 ) ;
2011-10-27 01:57:58 +02:00
}
if ( work . nwait = = work . nproc )
return nil ;
2012-10-23 06:31:11 +02:00
if ( i < 10 ) {
m - > gcstats . nprocyield + + ;
2011-10-27 01:57:58 +02:00
runtime_procyield ( 20 ) ;
2012-10-23 06:31:11 +02:00
} else if ( i < 20 ) {
m - > gcstats . nosyield + + ;
2011-10-27 01:57:58 +02:00
runtime_osyield ( ) ;
2012-10-23 06:31:11 +02:00
} else {
m - > gcstats . nsleep + + ;
2011-10-27 01:57:58 +02:00
runtime_usleep ( 100 ) ;
2012-10-23 06:31:11 +02:00
}
2011-03-17 00:05:44 +01:00
}
}
2011-10-27 01:57:58 +02:00
static Workbuf *
handoff ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 n ;
Workbuf * b1 ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
// Make new buffer with half of b's pointers.
b1 = getempty ( nil ) ;
n = b - > nobj / 2 ;
b - > nobj - = n ;
b1 - > nobj = n ;
runtime_memmove ( b1 - > obj , b - > obj + b - > nobj , n * sizeof b1 - > obj [ 0 ] ) ;
2012-10-23 06:31:11 +02:00
m - > gcstats . nhandoff + + ;
m - > gcstats . nhandoffcnt + = n ;
2011-10-27 01:57:58 +02:00
// Put b on full list - let first half of b get stolen.
2012-10-23 06:31:11 +02:00
runtime_lfstackpush ( & work . full , & b - > node ) ;
2011-10-27 01:57:58 +02:00
return b1 ;
}
2011-11-28 06:45:49 +01:00
static void
2012-12-22 02:15:33 +01:00
addroot ( Obj obj )
2012-10-23 06:31:11 +02:00
{
uint32 cap ;
2012-12-22 02:15:33 +01:00
Obj * new ;
2012-10-23 06:31:11 +02:00
if ( work . nroot > = work . rootcap ) {
2012-12-22 02:15:33 +01:00
cap = PageSize / sizeof ( Obj ) ;
2012-10-23 06:31:11 +02:00
if ( cap < 2 * work . rootcap )
cap = 2 * work . rootcap ;
2013-11-06 20:49:01 +01:00
new = ( Obj * ) runtime_SysAlloc ( cap * sizeof ( Obj ) , & mstats . gc_sys ) ;
2013-07-16 08:54:42 +02:00
if ( new = = nil )
runtime_throw ( " runtime: cannot allocate memory " ) ;
2012-10-23 06:31:11 +02:00
if ( work . roots ! = nil ) {
2012-12-22 02:15:33 +01:00
runtime_memmove ( new , work . roots , work . rootcap * sizeof ( Obj ) ) ;
2013-11-06 20:49:01 +01:00
runtime_SysFree ( work . roots , work . rootcap * sizeof ( Obj ) , & mstats . gc_sys ) ;
2012-10-23 06:31:11 +02:00
}
work . roots = new ;
work . rootcap = cap ;
}
2012-12-22 02:15:33 +01:00
work . roots [ work . nroot ] = obj ;
2012-10-23 06:31:11 +02:00
work . nroot + + ;
}
static void
addstackroots ( G * gp )
2011-11-28 06:45:49 +01:00
{
# ifdef USING_SPLIT_STACK
M * mp ;
void * sp ;
size_t spsize ;
void * next_segment ;
void * next_sp ;
void * initial_sp ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
sp = __splitstack_find ( nil , nil , & spsize , & next_segment ,
& next_sp , & initial_sp ) ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
// The exception is that if the goroutine is about to enter or might
// have just exited a system call, it may be executing code such
// as schedlock and may have needed to start a new stack segment.
// Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot.
if ( gp - > gcstack ! = nil ) {
sp = gp - > gcstack ;
spsize = gp - > gcstack_size ;
next_segment = gp - > gcnext_segment ;
next_sp = gp - > gcnext_sp ;
initial_sp = gp - > gcinitial_sp ;
} else {
sp = __splitstack_find_context ( & gp - > stack_context [ 0 ] ,
& spsize , & next_segment ,
& next_sp , & initial_sp ) ;
}
}
if ( sp ! = nil ) {
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
while ( ( sp = __splitstack_find ( next_segment , next_sp ,
& spsize , & next_segment ,
& next_sp , & initial_sp ) ) ! = nil )
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
}
# else
M * mp ;
byte * bottom ;
byte * top ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
bottom = ( byte * ) & gp ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
bottom = ( byte * ) gp - > gcnext_sp ;
if ( bottom = = nil )
return ;
}
top = ( byte * ) gp - > gcinitial_sp + gp - > gcstack_size ;
if ( top > bottom )
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { bottom , top - bottom , 0 } ) ;
2011-11-28 06:45:49 +01:00
else
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { top , bottom - top , 0 } ) ;
2011-11-28 06:45:49 +01:00
# endif
}
2010-12-03 05:34:57 +01:00
static void
2012-10-23 06:31:11 +02:00
addfinroots ( void * v )
2010-12-03 05:34:57 +01:00
{
uintptr size ;
2013-07-16 08:54:42 +02:00
void * base ;
2010-12-03 05:34:57 +01:00
size = 0 ;
2013-07-16 08:54:42 +02:00
if ( ! runtime_mlookup ( v , ( byte * * ) & base , & size , nil ) | | ! runtime_blockspecial ( base ) )
2010-12-03 05:34:57 +01:00
runtime_throw ( " mark - finalizer inconsistency " ) ;
2011-03-17 00:05:44 +01:00
2010-12-03 05:34:57 +01:00
// do not mark the finalizer block itself. just mark the things it points at.
2013-07-16 08:54:42 +02:00
addroot ( ( Obj ) { base , size , 0 } ) ;
2010-12-03 05:34:57 +01:00
}
static struct root_list * roots ;
void
__go_register_gc_roots ( struct root_list * r )
{
// FIXME: This needs locking if multiple goroutines can call
// dlopen simultaneously.
r - > next = roots ;
roots = r ;
}
static void
2012-10-23 06:31:11 +02:00
addroots ( void )
2010-12-03 05:34:57 +01:00
{
struct root_list * pl ;
2011-11-28 06:45:49 +01:00
G * gp ;
2011-10-27 01:57:58 +02:00
FinBlock * fb ;
2012-10-23 06:31:11 +02:00
MSpan * s , * * allspans ;
uint32 spanidx ;
work . nroot = 0 ;
2010-12-03 05:34:57 +01:00
2011-11-28 06:45:49 +01:00
// mark data+bss.
2010-12-03 05:34:57 +01:00
for ( pl = roots ; pl ! = nil ; pl = pl - > next ) {
struct root * pr = & pl - > roots [ 0 ] ;
while ( 1 ) {
void * decl = pr - > decl ;
if ( decl = = nil )
break ;
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { decl , pr - > size , 0 } ) ;
2010-12-03 05:34:57 +01:00
pr + + ;
}
}
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { ( byte * ) & runtime_m0 , sizeof runtime_m0 , 0 } ) ;
addroot ( ( Obj ) { ( byte * ) & runtime_g0 , sizeof runtime_g0 , 0 } ) ;
addroot ( ( Obj ) { ( byte * ) & runtime_allg , sizeof runtime_allg , 0 } ) ;
addroot ( ( Obj ) { ( byte * ) & runtime_allm , sizeof runtime_allm , 0 } ) ;
2013-07-16 08:54:42 +02:00
addroot ( ( Obj ) { ( byte * ) & runtime_allp , sizeof runtime_allp , 0 } ) ;
runtime_proc_scan ( addroot ) ;
2012-10-23 06:31:11 +02:00
runtime_MProf_Mark ( addroot ) ;
runtime_time_scan ( addroot ) ;
2013-11-14 21:15:04 +01:00
runtime_netpoll_scan ( addroot ) ;
2012-10-23 06:31:11 +02:00
// MSpan.types
2013-11-06 20:49:01 +01:00
allspans = runtime_mheap . allspans ;
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
2012-10-23 06:31:11 +02:00
s = allspans [ spanidx ] ;
if ( s - > state = = MSpanInUse ) {
2013-07-16 08:54:42 +02:00
// The garbage collector ignores type pointers stored in MSpan.types:
// - Compiler-generated types are stored outside of heap.
// - The reflect package has runtime-generated types cached in its data structures.
// The garbage collector relies on finding the references via that cache.
2012-10-23 06:31:11 +02:00
switch ( s - > types . compression ) {
case MTypes_Empty :
case MTypes_Single :
break ;
case MTypes_Words :
case MTypes_Bytes :
2013-07-16 08:54:42 +02:00
markonly ( ( byte * ) s - > types . data ) ;
2012-10-23 06:31:11 +02:00
break ;
}
}
}
2010-12-03 05:34:57 +01:00
2012-12-22 02:15:33 +01:00
// stacks
2011-11-28 06:45:49 +01:00
for ( gp = runtime_allg ; gp ! = nil ; gp = gp - > alllink ) {
switch ( gp - > status ) {
default :
runtime_printf ( " unexpected G.status %d \n " , gp - > status ) ;
runtime_throw ( " mark - bad status " ) ;
case Gdead :
break ;
case Grunning :
2013-11-06 20:49:01 +01:00
runtime_throw ( " mark - world not stopped " ) ;
2011-11-28 06:45:49 +01:00
case Grunnable :
case Gsyscall :
case Gwaiting :
2012-10-23 06:31:11 +02:00
addstackroots ( gp ) ;
2011-11-28 06:45:49 +01:00
break ;
}
}
2010-12-03 05:34:57 +01:00
2012-10-23 06:31:11 +02:00
runtime_walkfintab ( addfinroots , addroot ) ;
2011-10-27 01:57:58 +02:00
for ( fb = allfin ; fb ; fb = fb - > alllink )
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { ( byte * ) fb - > fin , fb - > cnt * sizeof ( fb - > fin [ 0 ] ) , 0 } ) ;
2011-10-27 01:57:58 +02:00
2012-12-22 02:15:33 +01:00
addroot ( ( Obj ) { ( byte * ) & work , sizeof work , 0 } ) ;
2010-12-03 05:34:57 +01:00
}
2014-06-05 01:15:33 +02:00
static void
addfreelists ( void )
{
int32 i ;
P * p , * * pp ;
MCache * c ;
MLink * m ;
// Mark objects in the MCache of each P so we don't collect them.
for ( pp = runtime_allp ; ( p = * pp ) ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
for ( i = 0 ; i < NumSizeClasses ; i + + ) {
for ( m = c - > list [ i ] . list ; m ! = nil ; m = m - > next ) {
markonly ( m ) ;
}
}
}
// Note: the sweeper will mark objects in each span's freelist.
}
2011-10-27 01:57:58 +02:00
static bool
handlespecial ( byte * p , uintptr size )
{
2013-06-19 01:49:49 +02:00
FuncVal * fn ;
2011-10-27 01:57:58 +02:00
const struct __go_func_type * ft ;
2013-11-06 20:49:01 +01:00
const struct __go_ptr_type * ot ;
2011-10-27 01:57:58 +02:00
FinBlock * block ;
Finalizer * f ;
2013-11-06 20:49:01 +01:00
if ( ! runtime_getfinalizer ( p , true , & fn , & ft , & ot ) ) {
2011-10-27 01:57:58 +02:00
runtime_setblockspecial ( p , false ) ;
runtime_MProf_Free ( p , size ) ;
return false ;
}
runtime_lock ( & finlock ) ;
if ( finq = = nil | | finq - > cnt = = finq - > cap ) {
if ( finc = = nil ) {
2013-11-06 20:49:01 +01:00
finc = runtime_persistentalloc ( PageSize , 0 , & mstats . gc_sys ) ;
2011-10-27 01:57:58 +02:00
finc - > cap = ( PageSize - sizeof ( FinBlock ) ) / sizeof ( Finalizer ) + 1 ;
finc - > alllink = allfin ;
allfin = finc ;
}
block = finc ;
finc = block - > next ;
block - > next = finq ;
finq = block ;
}
f = & finq - > fin [ finq - > cnt ] ;
finq - > cnt + + ;
f - > fn = fn ;
f - > ft = ft ;
2013-11-06 20:49:01 +01:00
f - > ot = ot ;
2011-10-27 01:57:58 +02:00
f - > arg = p ;
2012-10-23 06:31:11 +02:00
runtime_unlock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
return true ;
}
// Sweep frees or collects finalizers for blocks not marked in the mark phase.
2011-03-17 00:05:44 +01:00
// It clears the mark bits in preparation for the next GC round.
2010-12-03 05:34:57 +01:00
static void
2012-10-23 06:31:11 +02:00
sweepspan ( ParFor * desc , uint32 idx )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2011-03-17 00:05:44 +01:00
int32 cl , n , npages ;
2014-06-05 01:15:33 +02:00
uintptr size , off , * bitp , shift ;
2010-12-03 05:34:57 +01:00
byte * p ;
MCache * c ;
2011-10-27 01:57:58 +02:00
byte * arena_start ;
2012-10-23 06:31:11 +02:00
MLink head , * end ;
int32 nfree ;
byte * type_data ;
byte compression ;
uintptr type_data_inc ;
MSpan * s ;
2014-06-05 01:15:33 +02:00
MLink * x ;
2011-10-27 01:57:58 +02:00
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2012-10-23 06:31:11 +02:00
USED ( & desc ) ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . allspans [ idx ] ;
2012-10-23 06:31:11 +02:00
if ( s - > state ! = MSpanInUse )
return ;
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-10-23 06:31:11 +02:00
p = ( byte * ) ( s - > start < < PageShift ) ;
cl = s - > sizeclass ;
size = s - > elemsize ;
if ( cl = = 0 ) {
n = 1 ;
} else {
// Chunk full of small blocks.
npages = runtime_class_to_allocnpages [ cl ] ;
n = ( npages < < PageShift ) / size ;
}
nfree = 0 ;
end = & head ;
c = m - > mcache ;
2014-06-05 01:15:33 +02:00
// mark any free objects in this span so we don't collect them
for ( x = s - > freelist ; x ! = nil ; x = x - > next ) {
// This is markonly(x) but faster because we don't need
// atomic access and we're guaranteed to be pointing at
// the head of a valid object.
off = ( uintptr * ) x - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
* bitp | = bitMarked < < shift ;
}
2012-10-23 06:31:11 +02:00
type_data = ( byte * ) s - > types . data ;
type_data_inc = sizeof ( uintptr ) ;
compression = s - > types . compression ;
switch ( compression ) {
case MTypes_Bytes :
type_data + = 8 * sizeof ( uintptr ) ;
type_data_inc = 1 ;
break ;
}
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
// Sweep through n objects of given size starting at p.
// This thread owns the span now, so it can manipulate
// the block bitmap without atomic operations.
for ( ; n > 0 ; n - - , p + = size , type_data + = type_data_inc ) {
uintptr off , * bitp , shift , bits ;
2010-12-03 05:34:57 +01:00
2012-10-23 06:31:11 +02:00
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
2012-03-02 21:01:37 +01:00
2012-10-23 06:31:11 +02:00
if ( ( bits & bitAllocated ) = = 0 )
2011-03-17 00:05:44 +01:00
continue ;
2012-10-23 06:31:11 +02:00
if ( ( bits & bitMarked ) ! = 0 ) {
if ( DebugMark ) {
if ( ! ( bits & bitSpecial ) )
runtime_printf ( " found spurious mark on %p \n " , p ) ;
* bitp & = ~ ( bitSpecial < < shift ) ;
}
* bitp & = ~ ( bitMarked < < shift ) ;
continue ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
// Special means it has a finalizer or is being profiled.
// In DebugMark mode, the bit has been coopted so
// we have to assume all blocks are special.
if ( DebugMark | | ( bits & bitSpecial ) ! = 0 ) {
if ( handlespecial ( p , size ) )
2011-03-17 00:05:44 +01:00
continue ;
2012-10-23 06:31:11 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Clear mark, scan, and special bits.
* bitp & = ~ ( ( bitScan | bitMarked | bitSpecial ) < < shift ) ;
2011-03-17 00:05:44 +01:00
2012-10-23 06:31:11 +02:00
if ( cl = = 0 ) {
// Free large span.
runtime_unmarkspan ( p , 1 < < PageShift ) ;
2013-07-16 08:54:42 +02:00
* ( uintptr * ) p = ( uintptr ) 0xdeaddeaddeaddeadll ; // needs zeroing
2014-06-05 01:15:33 +02:00
if ( runtime_debug . efence )
runtime_SysFree ( p , size , & mstats . gc_sys ) ;
else
runtime_MHeap_Free ( & runtime_mheap , s , 1 ) ;
2013-11-06 20:49:01 +01:00
c - > local_nlargefree + + ;
c - > local_largefree + = size ;
2012-10-23 06:31:11 +02:00
} else {
// Free small object.
switch ( compression ) {
case MTypes_Words :
* ( uintptr * ) type_data = 0 ;
break ;
case MTypes_Bytes :
* ( byte * ) type_data = 0 ;
break ;
}
if ( size > sizeof ( uintptr ) )
2013-07-16 08:54:42 +02:00
( ( uintptr * ) p ) [ 1 ] = ( uintptr ) 0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed"
2012-10-23 06:31:11 +02:00
end - > next = ( MLink * ) p ;
end = ( MLink * ) p ;
nfree + + ;
2010-12-03 05:34:57 +01:00
}
}
2012-10-23 06:31:11 +02:00
if ( nfree ) {
2013-11-06 20:49:01 +01:00
c - > local_nsmallfree [ cl ] + = nfree ;
2012-10-23 06:31:11 +02:00
c - > local_cachealloc - = nfree * size ;
2013-11-06 20:49:01 +01:00
runtime_MCentral_FreeSpan ( & runtime_mheap . central [ cl ] , s , nfree , head . next , end ) ;
2012-10-23 06:31:11 +02:00
}
2010-12-03 05:34:57 +01:00
}
2012-11-21 08:03:38 +01:00
static void
dumpspan ( uint32 idx )
{
int32 sizeclass , n , npages , i , column ;
uintptr size ;
byte * p ;
byte * arena_start ;
MSpan * s ;
bool allocated , special ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . allspans [ idx ] ;
2012-11-21 08:03:38 +01:00
if ( s - > state ! = MSpanInUse )
return ;
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-11-21 08:03:38 +01:00
p = ( byte * ) ( s - > start < < PageShift ) ;
sizeclass = s - > sizeclass ;
size = s - > elemsize ;
if ( sizeclass = = 0 ) {
n = 1 ;
} else {
npages = runtime_class_to_allocnpages [ sizeclass ] ;
n = ( npages < < PageShift ) / size ;
}
runtime_printf ( " %p .. %p: \n " , p , p + n * size ) ;
column = 0 ;
for ( ; n > 0 ; n - - , p + = size ) {
uintptr off , * bitp , shift , bits ;
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
allocated = ( ( bits & bitAllocated ) ! = 0 ) ;
special = ( ( bits & bitSpecial ) ! = 0 ) ;
for ( i = 0 ; ( uint32 ) i < size ; i + = sizeof ( void * ) ) {
if ( column = = 0 ) {
runtime_printf ( " \t " ) ;
}
if ( i = = 0 ) {
runtime_printf ( allocated ? " ( " : " [ " ) ;
runtime_printf ( special ? " @ " : " " ) ;
runtime_printf ( " %p: " , p + i ) ;
} else {
runtime_printf ( " " ) ;
}
runtime_printf ( " %p " , * ( void * * ) ( p + i ) ) ;
if ( i + sizeof ( void * ) > = size ) {
runtime_printf ( allocated ? " ) " : " ] " ) ;
}
column + + ;
if ( column = = 8 ) {
runtime_printf ( " \n " ) ;
column = 0 ;
}
}
}
runtime_printf ( " \n " ) ;
}
// A debugging function to dump the contents of memory
void
runtime_memorydump ( void )
{
uint32 spanidx ;
2013-11-06 20:49:01 +01:00
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
2012-11-21 08:03:38 +01:00
dumpspan ( spanidx ) ;
}
}
2012-12-13 00:13:29 +01:00
2011-10-27 01:57:58 +02:00
void
runtime_gchelper ( void )
{
2014-03-03 21:14:52 +01:00
uint32 nproc ;
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
// parallel mark for over gc roots
runtime_parfordo ( work . markfor ) ;
2012-12-22 02:15:33 +01:00
2012-10-23 06:31:11 +02:00
// help other threads scan secondary blocks
2012-12-22 02:15:33 +01:00
scanblock ( nil , nil , 0 , true ) ;
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
if ( DebugMark ) {
// wait while the main thread executes mark(debug_scanblock)
while ( runtime_atomicload ( & work . debugmarkdone ) = = 0 )
runtime_usleep ( 10 ) ;
}
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
runtime_parfordo ( work . sweepfor ) ;
2013-07-16 08:54:42 +02:00
bufferList [ runtime_m ( ) - > helpgc ] . busy = 0 ;
2014-03-03 21:14:52 +01:00
nproc = work . nproc ; // work.nproc can change right after we increment work.ndone
if ( runtime_xadd ( & work . ndone , + 1 ) = = nproc - 1 )
2011-10-27 01:57:58 +02:00
runtime_notewakeup ( & work . alldone ) ;
}
2013-07-16 08:54:42 +02:00
# define GcpercentUnknown (-2)
2010-12-03 05:34:57 +01:00
// Initialized from $GOGC. GOGC=off means no gc.
//
// Next gc is after we've allocated an extra amount of
// memory proportional to the amount already in use.
// If gcpercent=100 and we're using 4M, we'll gc again
// when we get to 8M. This keeps the gc cost in linear
// proportion to the allocation cost. Adjusting gcpercent
// just changes the linear constant (and also the amount of
// extra memory used).
2013-07-16 08:54:42 +02:00
static int32 gcpercent = GcpercentUnknown ;
2011-11-28 06:45:49 +01:00
static void
2013-11-06 20:49:01 +01:00
cachestats ( void )
{
MCache * c ;
P * p , * * pp ;
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
runtime_purgecachedstats ( c ) ;
}
}
static void
updatememstats ( GCStats * stats )
2011-11-28 06:45:49 +01:00
{
2012-12-22 02:15:33 +01:00
M * mp ;
2013-11-06 20:49:01 +01:00
MSpan * s ;
2011-11-28 06:45:49 +01:00
MCache * c ;
2013-07-16 08:54:42 +02:00
P * p , * * pp ;
2011-11-28 06:45:49 +01:00
uint32 i ;
2013-11-06 20:49:01 +01:00
uint64 stacks_inuse , smallfree ;
2012-10-23 06:31:11 +02:00
uint64 * src , * dst ;
2011-11-28 06:45:49 +01:00
2012-10-23 06:31:11 +02:00
if ( stats )
runtime_memclr ( ( byte * ) stats , sizeof ( * stats ) ) ;
2011-11-28 06:45:49 +01:00
stacks_inuse = 0 ;
2012-12-22 02:15:33 +01:00
for ( mp = runtime_allm ; mp ; mp = mp - > alllink ) {
2013-07-16 08:54:42 +02:00
//stacks_inuse += mp->stackinuse*FixedStack;
2012-10-23 06:31:11 +02:00
if ( stats ) {
2012-12-22 02:15:33 +01:00
src = ( uint64 * ) & mp - > gcstats ;
2012-10-23 06:31:11 +02:00
dst = ( uint64 * ) stats ;
for ( i = 0 ; i < sizeof ( * stats ) / sizeof ( uint64 ) ; i + + )
dst [ i ] + = src [ i ] ;
2012-12-22 02:15:33 +01:00
runtime_memclr ( ( byte * ) & mp - > gcstats , sizeof ( mp - > gcstats ) ) ;
2012-10-23 06:31:11 +02:00
}
2013-07-16 08:54:42 +02:00
}
2013-11-06 20:49:01 +01:00
mstats . stacks_inuse = stacks_inuse ;
mstats . mcache_inuse = runtime_mheap . cachealloc . inuse ;
mstats . mspan_inuse = runtime_mheap . spanalloc . inuse ;
mstats . sys = mstats . heap_sys + mstats . stacks_sys + mstats . mspan_sys +
mstats . mcache_sys + mstats . buckhash_sys + mstats . gc_sys + mstats . other_sys ;
// Calculate memory allocator stats.
// During program execution we only count number of frees and amount of freed memory.
// Current number of alive object in the heap and amount of alive heap memory
// are calculated by scanning all spans.
// Total number of mallocs is calculated as number of frees plus number of alive objects.
// Similarly, total amount of allocated memory is calculated as amount of freed memory
// plus amount of alive heap memory.
mstats . alloc = 0 ;
mstats . total_alloc = 0 ;
mstats . nmalloc = 0 ;
mstats . nfree = 0 ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . by_size [ i ] . nmalloc = 0 ;
mstats . by_size [ i ] . nfree = 0 ;
}
// Flush MCache's to MCentral.
2013-07-16 08:54:42 +02:00
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
2013-11-06 20:49:01 +01:00
runtime_MCache_ReleaseAll ( c ) ;
}
// Aggregate local stats.
cachestats ( ) ;
// Scan all spans and count number of alive objects.
for ( i = 0 ; i < runtime_mheap . nspan ; i + + ) {
s = runtime_mheap . allspans [ i ] ;
if ( s - > state ! = MSpanInUse )
continue ;
if ( s - > sizeclass = = 0 ) {
mstats . nmalloc + + ;
mstats . alloc + = s - > elemsize ;
} else {
mstats . nmalloc + = s - > ref ;
mstats . by_size [ s - > sizeclass ] . nmalloc + = s - > ref ;
mstats . alloc + = s - > ref * s - > elemsize ;
2011-11-28 06:45:49 +01:00
}
}
2013-11-06 20:49:01 +01:00
// Aggregate by size class.
smallfree = 0 ;
mstats . nfree = runtime_mheap . nlargefree ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . nfree + = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nfree = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nmalloc + = runtime_mheap . nsmallfree [ i ] ;
smallfree + = runtime_mheap . nsmallfree [ i ] * runtime_class_to_size [ i ] ;
}
mstats . nmalloc + = mstats . nfree ;
// Calculate derived stats.
mstats . total_alloc = mstats . alloc + runtime_mheap . largefree + smallfree ;
mstats . heap_alloc = mstats . alloc ;
mstats . heap_objects = mstats . nmalloc - mstats . nfree ;
2011-11-28 06:45:49 +01:00
}
2012-12-13 00:13:29 +01:00
// Structure of arguments passed to function gc().
2013-11-06 20:49:01 +01:00
// This allows the arguments to be passed via runtime_mcall.
2012-12-13 00:13:29 +01:00
struct gc_args
{
2013-11-06 20:49:01 +01:00
int64 start_time ; // start time of GC in ns (just before stoptheworld)
2012-12-13 00:13:29 +01:00
} ;
static void gc ( struct gc_args * args ) ;
2013-11-06 20:49:01 +01:00
static void mgc ( G * gp ) ;
2012-12-13 00:13:29 +01:00
2013-07-16 08:54:42 +02:00
static int32
readgogc ( void )
{
const byte * p ;
p = runtime_getenv ( " GOGC " ) ;
if ( p = = nil | | p [ 0 ] = = ' \0 ' )
return 100 ;
if ( runtime_strcmp ( ( const char * ) p , " off " ) = = 0 )
return - 1 ;
return runtime_atoi ( p ) ;
}
2010-12-03 05:34:57 +01:00
void
2011-11-28 06:45:49 +01:00
runtime_gc ( int32 force )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2013-11-06 20:49:01 +01:00
G * g ;
struct gc_args a ;
int32 i ;
2012-10-23 06:31:11 +02:00
// The atomic operations are not atomic if the uint64s
// are not aligned on uint64 boundaries. This has been
// a problem in the past.
if ( ( ( ( uintptr ) & work . empty ) & 7 ) ! = 0 )
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2013-07-16 08:54:42 +02:00
if ( ( ( ( uintptr ) & work . full ) & 7 ) ! = 0 )
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2010-12-03 05:34:57 +01:00
2012-02-10 16:52:37 +01:00
// Make sure all registers are saved on stack so that
// scanstack sees them.
__builtin_unwind_init ( ) ;
2010-12-03 05:34:57 +01:00
// The gc is turned off (via enablegc) until
// the bootstrap has completed.
// Also, malloc gets called in the guts
// of a number of libraries that might be
// holding locks. To avoid priority inversion
// problems, don't bother trying to run gc
// while holding a lock. The next mallocgc
// without a lock will do the gc instead.
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2013-11-06 20:49:01 +01:00
if ( ! mstats . enablegc | | runtime_g ( ) = = m - > g0 | | m - > locks > 0 | | runtime_panicking )
2010-12-03 05:34:57 +01:00
return ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown ) { // first time through
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
runtime_unlock ( & runtime_mheap ) ;
2010-12-03 05:34:57 +01:00
}
if ( gcpercent < 0 )
return ;
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
if ( ! force & & mstats . heap_alloc < mstats . next_gc ) {
// typically threads which lost the race to grab
// worldsema exit here when gc is done.
runtime_semrelease ( & runtime_worldsema ) ;
return ;
}
2012-12-13 00:13:29 +01:00
2013-11-06 20:49:01 +01:00
// Ok, we're doing it! Stop everybody else
a . start_time = runtime_nanotime ( ) ;
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2014-06-05 01:15:33 +02:00
clearpools ( ) ;
2013-11-06 20:49:01 +01:00
// Run gc on the g0 stack. We do this so that the g stack
// we're currently running on will no longer change. Cuts
// the root set down a bit (g0 stacks are not scanned, and
// we don't need to scan gc's internal state). Also an
// enabler for copyable stacks.
for ( i = 0 ; i < ( runtime_debug . gctrace > 1 ? 2 : 1 ) ; i + + ) {
// switch to g0, call gc(&a), then switch back
g = runtime_g ( ) ;
g - > param = & a ;
g - > status = Gwaiting ;
g - > waitreason = " garbage collection " ;
runtime_mcall ( mgc ) ;
// record a new start time in case we're going around again
a . start_time = runtime_nanotime ( ) ;
2012-12-13 00:13:29 +01:00
}
2013-11-06 20:49:01 +01:00
// all done
m - > gcing = 0 ;
m - > locks + + ;
runtime_semrelease ( & runtime_worldsema ) ;
runtime_starttheworld ( ) ;
m - > locks - - ;
// now that gc is done, kick off finalizer thread if needed
if ( finq ! = nil ) {
runtime_lock ( & finlock ) ;
// kick off or wake up goroutine to run queued finalizers
if ( fing = = nil )
fing = __go_go ( runfinq , nil ) ;
else if ( fingwait ) {
fingwait = 0 ;
runtime_ready ( fing ) ;
}
runtime_unlock ( & finlock ) ;
}
// give the queued finalizers, if any, a chance to run
runtime_gosched ( ) ;
}
static void
mgc ( G * gp )
{
gc ( gp - > param ) ;
gp - > param = nil ;
gp - > status = Grunning ;
runtime_gogo ( gp ) ;
2012-12-13 00:13:29 +01:00
}
static void
gc ( struct gc_args * args )
{
M * m ;
2013-01-29 21:52:43 +01:00
int64 t0 , t1 , t2 , t3 , t4 ;
2013-07-16 08:54:42 +02:00
uint64 heap0 , heap1 , obj0 , obj1 , ninstr ;
2012-12-13 00:13:29 +01:00
GCStats stats ;
2012-12-22 02:15:33 +01:00
M * mp ;
2012-12-13 00:13:29 +01:00
uint32 i ;
2013-01-29 21:52:43 +01:00
// Eface eface;
2012-12-13 00:13:29 +01:00
m = runtime_m ( ) ;
2013-11-06 20:49:01 +01:00
t0 = args - > start_time ;
2010-12-03 05:34:57 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_memclr ( ( byte * ) & gcstats , sizeof ( gcstats ) ) ;
2012-12-22 02:15:33 +01:00
for ( mp = runtime_allm ; mp ; mp = mp - > alllink )
2013-11-06 20:49:01 +01:00
runtime_settype_flush ( mp ) ;
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
heap0 = 0 ;
obj0 = 0 ;
2013-11-06 20:49:01 +01:00
if ( runtime_debug . gctrace ) {
updatememstats ( nil ) ;
2012-10-23 06:31:11 +02:00
heap0 = mstats . heap_alloc ;
obj0 = mstats . nmalloc - mstats . nfree ;
2011-10-27 01:57:58 +02:00
}
2012-10-23 06:31:11 +02:00
2012-12-22 02:15:33 +01:00
m - > locks + + ; // disable gc during mallocs in parforalloc
if ( work . markfor = = nil )
work . markfor = runtime_parforalloc ( MaxGcproc ) ;
if ( work . sweepfor = = nil )
work . sweepfor = runtime_parforalloc ( MaxGcproc ) ;
m - > locks - - ;
2013-01-29 21:52:43 +01:00
if ( itabtype = = nil ) {
// get C pointer to the Go type "itab"
// runtime_gc_itab_ptr(&eface);
// itabtype = ((PtrType*)eface.type)->elem;
}
2011-10-27 01:57:58 +02:00
work . nwait = 0 ;
work . ndone = 0 ;
2012-10-23 06:31:11 +02:00
work . debugmarkdone = 0 ;
work . nproc = runtime_gcprocs ( ) ;
addroots ( ) ;
2014-06-05 01:15:33 +02:00
addfreelists ( ) ;
2012-10-23 06:31:11 +02:00
runtime_parforsetup ( work . markfor , work . nproc , work . nroot , nil , false , markroot ) ;
2013-11-06 20:49:01 +01:00
runtime_parforsetup ( work . sweepfor , work . nproc , runtime_mheap . nspan , nil , true , sweepspan ) ;
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 ) {
runtime_noteclear ( & work . alldone ) ;
runtime_helpgc ( work . nproc ) ;
}
2011-10-27 01:57:58 +02:00
2013-01-29 21:52:43 +01:00
t1 = runtime_nanotime ( ) ;
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
runtime_parfordo ( work . markfor ) ;
2012-12-22 02:15:33 +01:00
scanblock ( nil , nil , 0 , true ) ;
2012-10-23 06:31:11 +02:00
if ( DebugMark ) {
for ( i = 0 ; i < work . nroot ; i + + )
debug_scanblock ( work . roots [ i ] . p , work . roots [ i ] . n ) ;
runtime_atomicstore ( & work . debugmarkdone , 1 ) ;
}
2013-01-29 21:52:43 +01:00
t2 = runtime_nanotime ( ) ;
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
runtime_parfordo ( work . sweepfor ) ;
2013-07-16 08:54:42 +02:00
bufferList [ m - > helpgc ] . busy = 0 ;
2013-01-29 21:52:43 +01:00
t3 = runtime_nanotime ( ) ;
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 )
runtime_notesleep ( & work . alldone ) ;
2013-11-06 20:49:01 +01:00
cachestats ( ) ;
mstats . next_gc = mstats . heap_alloc + mstats . heap_alloc * gcpercent / 100 ;
2011-03-17 00:05:44 +01:00
2013-01-29 21:52:43 +01:00
t4 = runtime_nanotime ( ) ;
mstats . last_gc = t4 ;
mstats . pause_ns [ mstats . numgc % nelem ( mstats . pause_ns ) ] = t4 - t0 ;
mstats . pause_total_ns + = t4 - t0 ;
2010-12-03 05:34:57 +01:00
mstats . numgc + + ;
if ( mstats . debuggc )
2013-01-29 21:52:43 +01:00
runtime_printf ( " pause %D \n " , t4 - t0 ) ;
2011-10-27 01:57:58 +02:00
2013-11-06 20:49:01 +01:00
if ( runtime_debug . gctrace ) {
updatememstats ( & stats ) ;
heap1 = mstats . heap_alloc ;
obj1 = mstats . nmalloc - mstats . nfree ;
stats . nprocyield + = work . sweepfor - > nprocyield ;
stats . nosyield + = work . sweepfor - > nosyield ;
stats . nsleep + = work . sweepfor - > nsleep ;
2012-10-23 06:31:11 +02:00
runtime_printf ( " gc%d(%d): %D+%D+%D ms, %D -> %D MB %D -> %D (%D-%D) objects, "
" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields \n " ,
2013-01-29 21:52:43 +01:00
mstats . numgc , work . nproc , ( t2 - t1 ) / 1000000 , ( t3 - t2 ) / 1000000 , ( t1 - t0 + t4 - t3 ) / 1000000 ,
2012-05-24 22:44:34 +02:00
heap0 > > 20 , heap1 > > 20 , obj0 , obj1 ,
2012-10-23 06:31:11 +02:00
mstats . nmalloc , mstats . nfree ,
stats . nhandoff , stats . nhandoffcnt ,
work . sweepfor - > nsteal , work . sweepfor - > nstealcnt ,
stats . nprocyield , stats . nosyield , stats . nsleep ) ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_printf ( " scan: %D bytes, %D objects, %D untyped, %D types from MSpan \n " ,
gcstats . nbytes , gcstats . obj . cnt , gcstats . obj . notype , gcstats . obj . typelookup ) ;
if ( gcstats . ptr . cnt ! = 0 )
runtime_printf ( " avg ptrbufsize: %D (%D/%D) \n " ,
gcstats . ptr . sum / gcstats . ptr . cnt , gcstats . ptr . sum , gcstats . ptr . cnt ) ;
if ( gcstats . obj . cnt ! = 0 )
runtime_printf ( " avg nobj: %D (%D/%D) \n " ,
gcstats . obj . sum / gcstats . obj . cnt , gcstats . obj . sum , gcstats . obj . cnt ) ;
runtime_printf ( " rescans: %D, %D bytes \n " , gcstats . rescan , gcstats . rescanbytes ) ;
runtime_printf ( " instruction counts: \n " ) ;
ninstr = 0 ;
for ( i = 0 ; i < nelem ( gcstats . instr ) ; i + + ) {
runtime_printf ( " \t %d: \t %D \n " , i , gcstats . instr [ i ] ) ;
ninstr + = gcstats . instr [ i ] ;
}
runtime_printf ( " \t total: \t %D \n " , ninstr ) ;
runtime_printf ( " putempty: %D, getfull: %D \n " , gcstats . putempty , gcstats . getfull ) ;
2013-11-06 20:49:01 +01:00
runtime_printf ( " markonly base lookup: bit %D word %D span %D \n " , gcstats . markonly . foundbit , gcstats . markonly . foundword , gcstats . markonly . foundspan ) ;
runtime_printf ( " flushptrbuf base lookup: bit %D word %D span %D \n " , gcstats . flushptrbuf . foundbit , gcstats . flushptrbuf . foundword , gcstats . flushptrbuf . foundspan ) ;
2013-07-16 08:54:42 +02:00
}
2011-03-17 00:05:44 +01:00
}
2012-10-23 06:31:11 +02:00
2012-03-02 21:01:37 +01:00
runtime_MProf_GC ( ) ;
2010-12-03 05:34:57 +01:00
}
2012-02-09 09:19:58 +01:00
void runtime_ReadMemStats ( MStats * )
2013-01-24 20:44:23 +01:00
__asm__ ( GOSYM_PREFIX " runtime.ReadMemStats " ) ;
2011-09-16 17:47:21 +02:00
void
2012-02-09 09:19:58 +01:00
runtime_ReadMemStats ( MStats * stats )
2011-09-16 17:47:21 +02:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2012-03-02 21:01:37 +01:00
// Have to acquire worldsema to stop the world,
2011-09-16 17:47:21 +02:00
// because stoptheworld can only be used by
// one goroutine at a time, and there might be
// a pending garbage collection already calling it.
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2011-09-16 17:47:21 +02:00
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2013-11-06 20:49:01 +01:00
updatememstats ( nil ) ;
2012-02-09 09:19:58 +01:00
* stats = mstats ;
2011-09-16 17:47:21 +02:00
m - > gcing = 0 ;
2013-11-06 20:49:01 +01:00
m - > locks + + ;
2012-03-02 21:01:37 +01:00
runtime_semrelease ( & runtime_worldsema ) ;
2012-10-23 06:31:11 +02:00
runtime_starttheworld ( ) ;
2013-11-06 20:49:01 +01:00
m - > locks - - ;
2011-09-16 17:47:21 +02:00
}
2013-07-16 08:54:42 +02:00
void runtime_debug_readGCStats ( Slice * )
__asm__ ( " runtime_debug.readGCStats " ) ;
void
runtime_debug_readGCStats ( Slice * pauses )
{
uint64 * p ;
uint32 i , n ;
// Calling code in runtime/debug should make the slice large enough.
if ( ( size_t ) pauses - > cap < nelem ( mstats . pause_ns ) + 3 )
runtime_throw ( " runtime: short slice passed to readGCStats " ) ;
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = ( uint64 * ) pauses - > array ;
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
n = mstats . numgc ;
if ( n > nelem ( mstats . pause_ns ) )
n = nelem ( mstats . pause_ns ) ;
// The pause buffer is circular. The most recent pause is at
// pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
// from there to go back farther in time. We deliver the times
// most recent first (in p[0]).
for ( i = 0 ; i < n ; i + + )
p [ i ] = mstats . pause_ns [ ( mstats . numgc - 1 - i ) % nelem ( mstats . pause_ns ) ] ;
p [ n ] = mstats . last_gc ;
p [ n + 1 ] = mstats . numgc ;
p [ n + 2 ] = mstats . pause_total_ns ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
pauses - > __count = n + 3 ;
}
intgo runtime_debug_setGCPercent ( intgo )
__asm__ ( " runtime_debug.setGCPercent " ) ;
intgo
runtime_debug_setGCPercent ( intgo in )
{
intgo out ;
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
out = gcpercent ;
if ( in < 0 )
in = - 1 ;
gcpercent = in ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
return out ;
}
static void
gchelperstart ( void )
{
M * m ;
m = runtime_m ( ) ;
if ( m - > helpgc < 0 | | m - > helpgc > = MaxGcproc )
runtime_throw ( " gchelperstart: bad m->helpgc " ) ;
if ( runtime_xchg ( & bufferList [ m - > helpgc ] . busy , 1 ) )
runtime_throw ( " gchelperstart: already busy " ) ;
2013-11-06 20:49:01 +01:00
if ( runtime_g ( ) ! = m - > g0 )
runtime_throw ( " gchelper not running on g0 stack " ) ;
2013-07-16 08:54:42 +02:00
}
2010-12-03 05:34:57 +01:00
static void
2011-11-28 06:45:49 +01:00
runfinq ( void * dummy __attribute__ ( ( unused ) ) )
2010-12-03 05:34:57 +01:00
{
2011-10-27 01:57:58 +02:00
Finalizer * f ;
FinBlock * fb , * next ;
uint32 i ;
2013-11-06 20:49:01 +01:00
Eface ef ;
Iface iface ;
2010-12-03 05:34:57 +01:00
for ( ; ; ) {
2013-11-06 20:49:01 +01:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
fb = finq ;
2010-12-03 05:34:57 +01:00
finq = nil ;
2011-10-27 01:57:58 +02:00
if ( fb = = nil ) {
2010-12-03 05:34:57 +01:00
fingwait = 1 ;
2013-11-06 20:49:01 +01:00
runtime_park ( runtime_unlock , & finlock , " finalizer wait " ) ;
2010-12-03 05:34:57 +01:00
continue ;
}
2013-11-06 20:49:01 +01:00
runtime_unlock ( & finlock ) ;
2012-11-21 08:03:38 +01:00
if ( raceenabled )
runtime_racefingo ( ) ;
2011-10-27 01:57:58 +02:00
for ( ; fb ; fb = next ) {
next = fb - > next ;
for ( i = 0 ; i < ( uint32 ) fb - > cnt ; i + + ) {
2013-11-06 20:49:01 +01:00
const Type * fint ;
2013-09-03 23:52:37 +02:00
void * param ;
2011-10-27 01:57:58 +02:00
f = & fb - > fin [ i ] ;
2013-11-06 20:49:01 +01:00
fint = ( ( const Type * * ) f - > ft - > __in . array ) [ 0 ] ;
if ( fint - > kind = = KindPtr ) {
// direct use of pointer
param = & f - > arg ;
} else if ( ( ( const InterfaceType * ) fint ) - > __methods . __count = = 0 ) {
// convert to empty interface
ef . type = ( const Type * ) f - > ot ;
ef . __object = f - > arg ;
param = & ef ;
} else {
// convert to interface with methods
iface . __methods = __go_convert_interface_2 ( ( const Type * ) fint ,
( const Type * ) f - > ot ,
1 ) ;
iface . __object = f - > arg ;
if ( iface . __methods = = nil )
runtime_throw ( " invalid type conversion in runfinq " ) ;
param = & iface ;
}
2013-09-03 23:52:37 +02:00
reflect_call ( f - > ft , f - > fn , 0 , 0 , & param , nil ) ;
2011-10-27 01:57:58 +02:00
f - > fn = nil ;
f - > arg = nil ;
2013-11-06 20:49:01 +01:00
f - > ot = nil ;
2011-10-27 01:57:58 +02:00
}
fb - > cnt = 0 ;
fb - > next = finc ;
finc = fb ;
2010-12-03 05:34:57 +01:00
}
runtime_gc ( 1 ) ; // trigger another gc to clean up the finalized objects, if possible
}
}
2011-03-17 00:05:44 +01:00
void
2014-06-05 01:15:33 +02:00
runtime_marknogc ( void * v )
2011-03-17 00:05:44 +01:00
{
uintptr * b , obits , bits , off , shift ;
2014-06-05 01:15:33 +02:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
for ( ; ; ) {
obits = * b ;
if ( ( obits > > shift & bitMask ) ! = bitAllocated )
runtime_throw ( " bad initial state for marknogc " ) ;
bits = ( obits & ~ ( bitAllocated < < shift ) ) | bitBlockBoundary < < shift ;
if ( runtime_gomaxprocs = = 1 ) {
* b = bits ;
break ;
} else {
// more than one goroutine is potentially running: use atomic op
if ( runtime_casp ( ( void * * ) b , ( void * ) obits , ( void * ) bits ) )
break ;
}
}
}
void
runtime_markscan ( void * v )
{
uintptr * b , obits , bits , off , shift ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
for ( ; ; ) {
obits = * b ;
2014-06-05 01:15:33 +02:00
if ( ( obits > > shift & bitMask ) ! = bitAllocated )
runtime_throw ( " bad initial state for markscan " ) ;
bits = obits | bitScan < < shift ;
2013-11-06 20:49:01 +01:00
if ( runtime_gomaxprocs = = 1 ) {
2011-03-17 00:05:44 +01:00
* b = bits ;
break ;
} else {
2011-09-16 17:47:21 +02:00
// more than one goroutine is potentially running: use atomic op
2011-03-17 00:05:44 +01:00
if ( runtime_casp ( ( void * * ) b , ( void * ) obits , ( void * ) bits ) )
break ;
}
}
}
// mark the block at v of size n as freed.
void
runtime_markfreed ( void * v , uintptr n )
{
uintptr * b , obits , bits , off , shift ;
2012-05-24 22:44:34 +02:00
if ( 0 )
2013-11-06 20:49:01 +01:00
runtime_printf ( " markfreed %p+%p \n " , v , n ) ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
runtime_throw ( " markfreed: bad pointer " ) ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
for ( ; ; ) {
obits = * b ;
2014-06-05 01:15:33 +02:00
// This could be a free of a gc-eligible object (bitAllocated + others) or
// a FlagNoGC object (bitBlockBoundary set). In either case, we revert to
// a simple no-scan allocated object because it is going on a free list.
bits = ( obits & ~ ( bitMask < < shift ) ) | ( bitAllocated < < shift ) ;
2013-11-06 20:49:01 +01:00
if ( runtime_gomaxprocs = = 1 ) {
2011-03-17 00:05:44 +01:00
* b = bits ;
break ;
} else {
2011-09-16 17:47:21 +02:00
// more than one goroutine is potentially running: use atomic op
2011-03-17 00:05:44 +01:00
if ( runtime_casp ( ( void * * ) b , ( void * ) obits , ( void * ) bits ) )
break ;
}
}
}
// check that the block at v of size n is marked freed.
void
runtime_checkfreed ( void * v , uintptr n )
{
uintptr * b , bits , off , shift ;
if ( ! runtime_checking )
return ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
return ; // not allocated, so okay
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
bits = * b > > shift ;
if ( ( bits & bitAllocated ) ! = 0 ) {
runtime_printf ( " checkfreed %p+%p: off=%p have=%p \n " ,
2012-05-24 22:44:34 +02:00
v , n , off , bits & bitMask ) ;
2011-03-17 00:05:44 +01:00
runtime_throw ( " checkfreed: not freed " ) ;
}
}
// mark the span of memory at v as having n blocks of the given size.
// if leftover is true, there is left over space at the end of the span.
void
runtime_markspan ( void * v , uintptr size , uintptr n , bool leftover )
{
2014-06-05 01:15:33 +02:00
uintptr * b , off , shift , i ;
2011-03-17 00:05:44 +01:00
byte * p ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + size * n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
2014-06-05 01:15:33 +02:00
if ( runtime_checking ) {
// bits should be all zero at the start
off = ( byte * ) v + size - runtime_mheap . arena_start ;
b = ( uintptr * ) ( runtime_mheap . arena_start - off / wordsPerBitmapWord ) ;
for ( i = 0 ; i < size / PtrSize / wordsPerBitmapWord ; i + + ) {
if ( b [ i ] ! = 0 )
runtime_throw ( " markspan: span bits not zero " ) ;
}
}
2011-03-17 00:05:44 +01:00
p = v ;
if ( leftover ) // mark a boundary just past end of last block too
n + + ;
for ( ; n - - > 0 ; p + = size ) {
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-06-05 01:15:33 +02:00
* b = ( * b & ~ ( bitMask < < shift ) ) | ( bitAllocated < < shift ) ;
2011-03-17 00:05:44 +01:00
}
}
// unmark the span of memory at v of length n bytes.
void
runtime_unmarkspan ( void * v , uintptr n )
{
uintptr * p , * b , off ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
p = v ;
2013-11-06 20:49:01 +01:00
off = p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
2011-03-17 00:05:44 +01:00
if ( off % wordsPerBitmapWord ! = 0 )
runtime_throw ( " markspan: unaligned pointer " ) ;
2013-11-06 20:49:01 +01:00
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
n / = PtrSize ;
if ( n % wordsPerBitmapWord ! = 0 )
runtime_throw ( " unmarkspan: unaligned length " ) ;
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
n / = wordsPerBitmapWord ;
while ( n - - > 0 )
* b - - = 0 ;
}
bool
runtime_blockspecial ( void * v )
{
uintptr * b , off , shift ;
2011-10-27 01:57:58 +02:00
if ( DebugMark )
return true ;
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ;
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
return ( * b & ( bitSpecial < < shift ) ) ! = 0 ;
}
void
2011-10-27 01:57:58 +02:00
runtime_setblockspecial ( void * v , bool s )
2011-03-17 00:05:44 +01:00
{
uintptr * b , off , shift , bits , obits ;
2011-10-27 01:57:58 +02:00
if ( DebugMark )
return ;
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ;
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
for ( ; ; ) {
obits = * b ;
2011-10-27 01:57:58 +02:00
if ( s )
bits = obits | ( bitSpecial < < shift ) ;
else
bits = obits & ~ ( bitSpecial < < shift ) ;
2013-11-06 20:49:01 +01:00
if ( runtime_gomaxprocs = = 1 ) {
2011-03-17 00:05:44 +01:00
* b = bits ;
break ;
} else {
2011-09-16 17:47:21 +02:00
// more than one goroutine is potentially running: use atomic op
2011-03-17 00:05:44 +01:00
if ( runtime_casp ( ( void * * ) b , ( void * ) obits , ( void * ) bits ) )
break ;
}
}
}
2011-10-27 01:57:58 +02:00
2011-03-17 00:05:44 +01:00
void
runtime_MHeap_MapBits ( MHeap * h )
{
2012-04-20 06:58:26 +02:00
size_t page_size ;
2011-03-17 00:05:44 +01:00
// Caller has added extra mappings to the arena.
// Add extra mappings of bitmap words as needed.
// We allocate extra bitmap pieces in chunks of bitmapChunk.
enum {
bitmapChunk = 8192
} ;
uintptr n ;
2011-10-27 01:57:58 +02:00
2011-03-17 00:05:44 +01:00
n = ( h - > arena_used - h - > arena_start ) / wordsPerBitmapWord ;
2013-11-06 20:49:01 +01:00
n = ROUND ( n , bitmapChunk ) ;
2011-03-17 00:05:44 +01:00
if ( h - > bitmap_mapped > = n )
return ;
2012-04-20 06:58:26 +02:00
page_size = getpagesize ( ) ;
n = ( n + page_size - 1 ) & ~ ( page_size - 1 ) ;
2013-11-06 20:49:01 +01:00
runtime_SysMap ( h - > arena_start - n , n - h - > bitmap_mapped , & mstats . gc_sys ) ;
2011-03-17 00:05:44 +01:00
h - > bitmap_mapped = n ;
}