2014-09-20 22:42:59 +02:00
// Copyright 2009 The Go Authors. All rights reserved.
2010-12-03 05:34:57 +01:00
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
2014-06-07 00:37:27 +02:00
// Garbage collector (GC).
//
// GC is:
// - mark&sweep
// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
// - parallel (up to MaxGcproc threads)
// - partially concurrent (mark is stop-the-world, while sweep is concurrent)
// - non-moving/non-compacting
// - full (non-partial)
//
// GC rate.
// Next GC is after we've allocated an extra amount of memory proportional to
// the amount already in use. The proportion is controlled by GOGC environment variable
// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
// (and also the amount of extra memory used).
//
// Concurrent sweep.
// The sweep phase proceeds concurrently with normal program execution.
// The heap is swept span-by-span both lazily (when a goroutine needs another span)
// and concurrently in a background goroutine (this helps programs that are not CPU bound).
// However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
// and so next_gc calculation is tricky and happens as follows.
// At the end of the stop-the-world phase next_gc is conservatively set based on total
// heap size; all spans are marked as "needs sweeping".
// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
// closer to the target value. However, this is not enough to avoid over-allocating memory.
// Consider that a goroutine wants to allocate a new span for a large object and
// there are no free swept spans, but there are small-object unswept spans.
// If the goroutine naively allocates a new span, it can surpass the yet-unknown
// target next_gc value. In order to prevent such cases (1) when a goroutine needs
// to allocate a new small-object span, it sweeps small-object spans for the same
// object size until it frees at least one object; (2) when a goroutine needs to
// allocate large-object span from heap, it sweeps spans until it frees at least
// that many pages into heap. Together these two measures ensure that we don't surpass
// target next_gc value by a large margin. There is an exception: if a goroutine sweeps
// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
// but there can still be other one-page unswept spans which could be combined into a two-page span.
// It's critical to ensure that no operations proceed on unswept spans (that would corrupt
// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
// When a goroutine explicitly frees an object or sets a finalizer, it ensures that
// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
// The finalizer goroutine is kicked off only when all spans are swept.
// When the next GC starts, it sweeps all not-yet-swept spans (if any).
2010-12-03 05:34:57 +01:00
2012-04-20 06:58:26 +02:00
# include <unistd.h>
2010-12-03 05:34:57 +01:00
# include "runtime.h"
2011-10-27 01:57:58 +02:00
# include "arch.h"
2010-12-03 05:34:57 +01:00
# include "malloc.h"
2012-12-22 02:15:33 +01:00
# include "mgc0.h"
2014-07-19 10:53:52 +02:00
# include "chan.h"
2013-01-29 21:52:43 +01:00
# include "go-type.h"
// Map gccgo field names to gc field names.
// Slice aka __go_open_array.
# define array __values
# define cap __capacity
// Iface aka __go_interface
# define tab __methods
2013-07-16 08:54:42 +02:00
// Hmap aka __go_map
typedef struct __go_map Hmap ;
2013-01-29 21:52:43 +01:00
// Type aka __go_type_descriptor
2013-07-16 08:54:42 +02:00
# define string __reflection
2013-01-29 21:52:43 +01:00
# define KindPtr GO_PTR
# define KindNoPointers GO_NO_POINTERS
2015-01-15 01:27:56 +01:00
# define kindMask GO_CODE_MASK
2013-01-29 21:52:43 +01:00
// PtrType aka __go_ptr_type
# define elem __element_type
2010-12-03 05:34:57 +01:00
2011-11-28 06:45:49 +01:00
# ifdef USING_SPLIT_STACK
extern void * __splitstack_find ( void * , void * , size_t * , void * * , void * * ,
void * * ) ;
extern void * __splitstack_find_context ( void * context [ 10 ] , size_t * , void * * ,
void * * , void * * ) ;
# endif
2010-12-03 05:34:57 +01:00
enum {
2011-03-17 00:05:44 +01:00
Debug = 0 ,
2013-07-16 08:54:42 +02:00
CollectStats = 0 ,
2014-06-07 00:37:27 +02:00
ConcurrentSweep = 1 ,
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
WorkbufSize = 16 * 1024 ,
FinBlockSize = 4 * 1024 ,
2012-12-22 02:15:33 +01:00
handoffThreshold = 4 ,
IntermediateBufferCapacity = 64 ,
2013-01-29 21:52:43 +01:00
// Bits in type information
PRECISE = 1 ,
LOOP = 2 ,
PC_BITS = PRECISE | LOOP ,
2013-11-06 20:49:01 +01:00
2014-06-07 00:37:27 +02:00
RootData = 0 ,
RootBss = 1 ,
RootFinalizers = 2 ,
RootSpanTypes = 3 ,
RootFlushCaches = 4 ,
RootCount = 5 ,
2010-12-03 05:34:57 +01:00
} ;
2014-06-07 00:37:27 +02:00
# define GcpercentUnknown (-2)
// Initialized from $GOGC. GOGC=off means no gc.
static int32 gcpercent = GcpercentUnknown ;
2014-07-19 10:53:52 +02:00
static FuncVal * poolcleanup ;
2014-06-05 01:15:33 +02:00
2014-07-19 10:53:52 +02:00
void sync_runtime_registerPoolCleanup ( FuncVal * )
__asm__ ( GOSYM_PREFIX " sync.runtime_registerPoolCleanup " ) ;
2014-06-05 01:15:33 +02:00
void
2014-07-19 10:53:52 +02:00
sync_runtime_registerPoolCleanup ( FuncVal * f )
2014-06-05 01:15:33 +02:00
{
2014-07-19 10:53:52 +02:00
poolcleanup = f ;
2014-06-05 01:15:33 +02:00
}
static void
clearpools ( void )
{
2014-06-07 00:37:27 +02:00
P * p , * * pp ;
MCache * c ;
// clear sync.Pool's
2014-07-19 10:53:52 +02:00
if ( poolcleanup ! = nil ) {
2015-01-16 23:58:53 +01:00
__builtin_call_with_static_chain ( poolcleanup - > fn ( ) ,
poolcleanup ) ;
2014-06-05 01:15:33 +02:00
}
2014-06-07 00:37:27 +02:00
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
// clear tinyalloc pool
c = p - > mcache ;
if ( c ! = nil ) {
c - > tiny = nil ;
c - > tinysize = 0 ;
}
// clear defer pools
p - > deferpool = nil ;
}
2014-06-05 01:15:33 +02:00
}
2012-03-02 21:01:37 +01:00
// Holding worldsema grants an M the right to try to stop the world.
// The procedure is:
//
// runtime_semacquire(&runtime_worldsema);
// m->gcing = 1;
// runtime_stoptheworld();
//
// ... do stuff ...
//
// m->gcing = 0;
// runtime_semrelease(&runtime_worldsema);
// runtime_starttheworld();
//
uint32 runtime_worldsema = 1 ;
2011-03-17 00:05:44 +01:00
typedef struct Workbuf Workbuf ;
struct Workbuf
2010-12-03 05:34:57 +01:00
{
2014-06-07 00:37:27 +02:00
# define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr))
2012-12-22 02:15:33 +01:00
LFNode node ; // must be first
2011-10-27 01:57:58 +02:00
uintptr nobj ;
2012-12-22 02:15:33 +01:00
Obj obj [ SIZE / sizeof ( Obj ) - 1 ] ;
uint8 _padding [ SIZE % sizeof ( Obj ) + sizeof ( Obj ) ] ;
# undef SIZE
2011-10-27 01:57:58 +02:00
} ;
typedef struct Finalizer Finalizer ;
struct Finalizer
{
2013-06-19 01:49:49 +02:00
FuncVal * fn ;
2011-10-27 01:57:58 +02:00
void * arg ;
const struct __go_func_type * ft ;
2014-09-04 00:56:09 +02:00
const PtrType * ot ;
2011-10-27 01:57:58 +02:00
} ;
typedef struct FinBlock FinBlock ;
struct FinBlock
{
FinBlock * alllink ;
FinBlock * next ;
int32 cnt ;
int32 cap ;
Finalizer fin [ 1 ] ;
2010-12-03 05:34:57 +01:00
} ;
2014-07-19 10:53:52 +02:00
static Lock finlock ; // protects the following variables
static FinBlock * finq ; // list of finalizers that are to be executed
static FinBlock * finc ; // cache of free blocks
static FinBlock * allfin ; // list of all blocks
bool runtime_fingwait ;
bool runtime_fingwake ;
2014-06-07 00:37:27 +02:00
static Lock gclock ;
2014-07-19 10:53:52 +02:00
static G * fing ;
2010-12-03 05:34:57 +01:00
2014-06-07 00:37:27 +02:00
static void runfinq ( void * ) ;
static void bgsweep ( void * ) ;
2011-03-17 00:05:44 +01:00
static Workbuf * getempty ( Workbuf * ) ;
static Workbuf * getfull ( Workbuf * ) ;
2011-10-27 01:57:58 +02:00
static void putempty ( Workbuf * ) ;
static Workbuf * handoff ( Workbuf * ) ;
2013-07-16 08:54:42 +02:00
static void gchelperstart ( void ) ;
2014-06-07 00:37:27 +02:00
static void flushallmcaches ( void ) ;
static void addstackroots ( G * gp , Workbuf * * wbufp ) ;
2011-10-27 01:57:58 +02:00
static struct {
2012-10-23 06:31:11 +02:00
uint64 full ; // lock-free list of full blocks
uint64 empty ; // lock-free list of empty blocks
byte pad0 [ CacheLineSize ] ; // prevents false-sharing between full/empty and nproc/nwait
2011-10-27 01:57:58 +02:00
uint32 nproc ;
2014-06-07 00:37:27 +02:00
int64 tstart ;
2011-10-27 01:57:58 +02:00
volatile uint32 nwait ;
volatile uint32 ndone ;
Note alldone ;
2012-10-23 06:31:11 +02:00
ParFor * markfor ;
2011-10-27 01:57:58 +02:00
Lock ;
byte * chunk ;
uintptr nchunk ;
2014-01-17 23:43:03 +01:00
} work __attribute__ ( ( aligned ( 8 ) ) ) ;
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR ,
2013-07-16 08:54:42 +02:00
GC_CHAN ,
GC_NUM_INSTR2
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static struct {
struct {
uint64 sum ;
uint64 cnt ;
} ptr ;
uint64 nbytes ;
struct {
uint64 sum ;
uint64 cnt ;
uint64 notype ;
uint64 typelookup ;
} obj ;
uint64 rescan ;
uint64 rescanbytes ;
uint64 instr [ GC_NUM_INSTR2 ] ;
uint64 putempty ;
uint64 getfull ;
2013-11-06 20:49:01 +01:00
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} flushptrbuf ;
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} markonly ;
2014-06-07 00:37:27 +02:00
uint32 nbgsweep ;
uint32 npausesweep ;
2013-07-16 08:54:42 +02:00
} gcstats ;
// markonly marks an object. It returns true if the object
// has been marked by this function, false otherwise.
// This function doesn't append the object to any buffer.
static bool
2014-07-19 10:53:52 +02:00
markonly ( const void * obj )
2013-07-16 08:54:42 +02:00
{
byte * p ;
2013-11-06 20:49:01 +01:00
uintptr * bitp , bits , shift , x , xbits , off , j ;
2013-07-16 08:54:42 +02:00
MSpan * s ;
PageID k ;
// Words outside the arena cannot be pointers.
2014-07-19 10:53:52 +02:00
if ( ( const byte * ) obj < runtime_mheap . arena_start | | ( const byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return false ;
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
2014-07-19 10:53:52 +02:00
obj = ( const void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
2013-07-16 08:54:42 +02:00
// Find bits for this word.
2014-07-19 10:53:52 +02:00
off = ( const uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
2013-11-06 20:49:01 +01:00
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
// Pointing at the beginning of a block?
2013-11-06 20:49:01 +01:00
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundbit , 1 ) ;
2013-07-16 08:54:42 +02:00
goto found ;
2013-11-06 20:49:01 +01:00
}
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
shift = j ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundword , 1 ) ;
goto found ;
}
}
2013-07-16 08:54:42 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) runtime_mheap . arena_start > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
2014-07-19 10:53:52 +02:00
if ( s = = nil | | k < s - > start | | ( const byte * ) obj > = s - > limit | | s - > state ! = MSpanInUse )
2013-07-16 08:54:42 +02:00
return false ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
uintptr size = s - > elemsize ;
2014-07-19 10:53:52 +02:00
int32 i = ( ( const byte * ) obj - p ) / size ;
2013-07-16 08:54:42 +02:00
obj = p + i * size ;
}
// Now that we know the object header, reload bits.
2014-07-19 10:53:52 +02:00
off = ( const uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
2013-11-06 20:49:01 +01:00
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundspan , 1 ) ;
2013-07-16 08:54:42 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
return false ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
return false ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
}
}
// The object is now marked
return true ;
}
// PtrTarget is a structure used by intermediate buffers.
2012-12-22 02:15:33 +01:00
// The intermediate buffers hold GC data before it
// is moved/flushed to the work buffer (Workbuf).
// The size of an intermediate buffer is very small,
// such as 32 or 64 elements.
2013-01-29 21:52:43 +01:00
typedef struct PtrTarget PtrTarget ;
2012-12-22 02:15:33 +01:00
struct PtrTarget
{
void * p ;
uintptr ti ;
} ;
2014-06-05 01:15:33 +02:00
typedef struct Scanbuf Scanbuf ;
struct Scanbuf
{
struct {
PtrTarget * begin ;
PtrTarget * end ;
PtrTarget * pos ;
} ptr ;
struct {
Obj * begin ;
Obj * end ;
Obj * pos ;
} obj ;
Workbuf * wbuf ;
Obj * wp ;
uintptr nobj ;
} ;
2013-01-29 21:52:43 +01:00
typedef struct BufferList BufferList ;
2012-12-22 02:15:33 +01:00
struct BufferList
{
2013-01-29 21:52:43 +01:00
PtrTarget ptrtarget [ IntermediateBufferCapacity ] ;
2013-07-16 08:54:42 +02:00
Obj obj [ IntermediateBufferCapacity ] ;
uint32 busy ;
byte pad [ CacheLineSize ] ;
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static BufferList bufferList [ MaxGcproc ] ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
static void enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj ) ;
2012-12-22 02:15:33 +01:00
// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
// while the work buffer contains blocks which have been marked
// and are prepared to be scanned by the garbage collector.
//
// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
//
// A simplified drawing explaining how the todo-list moves from a structure to another:
//
// scanblock
// (find pointers)
// Obj ------> PtrTarget (pointer targets)
// ↑ |
2013-07-16 08:54:42 +02:00
// | |
// `----------'
// flushptrbuf
// (find block start, mark and enqueue)
2010-12-03 05:34:57 +01:00
static void
2014-06-05 01:15:33 +02:00
flushptrbuf ( Scanbuf * sbuf )
2010-12-03 05:34:57 +01:00
{
2012-12-22 02:15:33 +01:00
byte * p , * arena_start , * obj ;
2013-01-29 21:52:43 +01:00
uintptr size , * bitp , bits , shift , j , x , xbits , off , nobj , ti , n ;
2011-03-17 00:05:44 +01:00
MSpan * s ;
PageID k ;
2012-12-22 02:15:33 +01:00
Obj * wp ;
2011-03-17 00:05:44 +01:00
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
PtrTarget * ptrbuf ;
2013-01-29 21:52:43 +01:00
PtrTarget * ptrbuf_end ;
2011-05-20 02:18:15 +02:00
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
ptrbuf = sbuf - > ptr . begin ;
ptrbuf_end = sbuf - > ptr . pos ;
n = ptrbuf_end - sbuf - > ptr . begin ;
sbuf - > ptr . pos = sbuf - > ptr . begin ;
2011-03-17 00:05:44 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . ptr . sum , n ) ;
runtime_xadd64 ( & gcstats . ptr . cnt , 1 ) ;
}
2012-12-22 02:15:33 +01:00
// If buffer is nearly full, get a new one.
if ( wbuf = = nil | | nobj + n > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
if ( n > = nelem ( wbuf - > obj ) )
runtime_throw ( " ptrbuf has to be smaller than WorkBuf " ) ;
2011-03-17 00:05:44 +01:00
}
2010-12-03 05:34:57 +01:00
2014-06-05 01:15:33 +02:00
while ( ptrbuf < ptrbuf_end ) {
obj = ptrbuf - > p ;
ti = ptrbuf - > ti ;
ptrbuf + + ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if ( Debug > 1 ) {
if ( obj < runtime_mheap . arena_start | | obj > = runtime_mheap . arena_used )
runtime_throw ( " object is outside of mheap " ) ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
if ( ( ( uintptr ) obj & ( ( uintptr ) PtrSize - 1 ) ) ! = 0 ) {
obj = ( void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
ti = 0 ;
}
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Find bits for this word.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Pointing at the beginning of a block?
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundbit , 1 ) ;
goto found ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
ti = 0 ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
obj = ( byte * ) obj - ( shift - j ) * PtrSize ;
shift = j ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . flushptrbuf . foundword , 1 ) ;
2011-03-17 00:05:44 +01:00
goto found ;
2013-11-06 20:49:01 +01:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
if ( s = = nil | | k < s - > start | | obj > = s - > limit | | s - > state ! = MSpanInUse )
continue ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
size = s - > elemsize ;
int32 i = ( ( byte * ) obj - p ) / size ;
obj = p + i * size ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Now that we know the object header, reload bits.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundspan , 1 ) ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
continue ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
goto continue_obj ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
2013-07-16 08:54:42 +02:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// If object has no pointers, don't need to scan further.
if ( ( bits & bitScan ) = = 0 )
continue ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = ( uintptr ) obj > > PageShift ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
PREFETCH ( obj ) ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
* wp = ( Obj ) { obj , s - > elemsize , ti } ;
wp + + ;
nobj + + ;
continue_obj : ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
2012-12-22 02:15:33 +01:00
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2012-12-22 02:15:33 +01:00
}
2013-07-16 08:54:42 +02:00
static void
2014-06-05 01:15:33 +02:00
flushobjbuf ( Scanbuf * sbuf )
2013-07-16 08:54:42 +02:00
{
uintptr nobj , off ;
Obj * wp , obj ;
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
Obj * objbuf ;
2013-07-16 08:54:42 +02:00
Obj * objbuf_end ;
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2013-07-16 08:54:42 +02:00
2014-06-05 01:15:33 +02:00
objbuf = sbuf - > obj . begin ;
objbuf_end = sbuf - > obj . pos ;
sbuf - > obj . pos = sbuf - > obj . begin ;
2013-07-16 08:54:42 +02:00
while ( objbuf < objbuf_end ) {
obj = * objbuf + + ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
continue ;
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
}
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2013-07-16 08:54:42 +02:00
}
2012-12-22 02:15:33 +01:00
// Program that scans the whole block and treats every block element as a potential pointer
static uintptr defaultProg [ 2 ] = { PtrSize , GC_DEFAULT_PTR } ;
2013-07-16 08:54:42 +02:00
// Hchan program
static uintptr chanProg [ 2 ] = { 0 , GC_CHAN } ;
2013-01-29 21:52:43 +01:00
// Local variables of a program fragment or loop
typedef struct Frame Frame ;
struct Frame {
uintptr count , elemsize , b ;
2014-09-04 00:56:09 +02:00
const uintptr * loop_or_ret ;
2013-01-29 21:52:43 +01:00
} ;
2013-07-16 08:54:42 +02:00
// Sanity check for the derived type info objti.
static void
checkptr ( void * obj , uintptr objti )
{
2014-09-04 00:56:09 +02:00
uintptr * pc1 , type , tisize , i , j , x ;
const uintptr * pc2 ;
2013-07-16 08:54:42 +02:00
byte * objstart ;
Type * t ;
MSpan * s ;
if ( ! Debug )
runtime_throw ( " checkptr is debug only " ) ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj < runtime_mheap . arena_start | | ( byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return ;
type = runtime_gettype ( obj ) ;
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
if ( t = = nil )
return ;
x = ( uintptr ) obj > > PageShift ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) ( runtime_mheap . arena_start ) > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
2013-07-16 08:54:42 +02:00
objstart = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass ! = 0 ) {
i = ( ( byte * ) obj - objstart ) / s - > elemsize ;
objstart + = i * s - > elemsize ;
}
tisize = * ( uintptr * ) objti ;
// Sanity check for object size: it should fit into the memory block.
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj + tisize > objstart + s - > elemsize ) {
runtime_printf ( " object of type '%S' at %p/%p does not fit in block %p/%p \n " ,
* t - > string , obj , tisize , objstart , s - > elemsize ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
2013-11-06 20:49:01 +01:00
}
2013-07-16 08:54:42 +02:00
if ( obj ! = objstart )
return ;
// If obj points to the beginning of the memory block,
// check type info as well.
if ( t - > string = = nil | |
// Gob allocates unsafe pointers for indirection.
( runtime_strcmp ( ( const char * ) t - > string - > str , ( const char * ) " unsafe.Pointer " ) & &
// Runtime and gc think differently about closures.
runtime_strstr ( ( const char * ) t - > string - > str , ( const char * ) " struct { F uintptr " ) ! = ( const char * ) t - > string - > str ) ) {
pc1 = ( uintptr * ) objti ;
2014-09-04 00:56:09 +02:00
pc2 = ( const uintptr * ) t - > __gc ;
2013-07-16 08:54:42 +02:00
// A simple best-effort check until first GC_END.
for ( j = 1 ; pc1 [ j ] ! = GC_END & & pc2 [ j ] ! = GC_END ; j + + ) {
if ( pc1 [ j ] ! = pc2 [ j ] ) {
2014-07-19 10:53:52 +02:00
runtime_printf ( " invalid gc type info for '%s', type info %p [%d]=%p, block info %p [%d]=%p \n " ,
t - > string ? ( const int8 * ) t - > string - > str : ( const int8 * ) " ? " , pc1 , ( int32 ) j , pc1 [ j ] , pc2 , ( int32 ) j , pc2 [ j ] ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
}
}
}
}
2012-12-22 02:15:33 +01:00
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
static void
2014-06-07 00:37:27 +02:00
scanblock ( Workbuf * wbuf , bool keepworking )
2012-12-22 02:15:33 +01:00
{
byte * b , * arena_start , * arena_used ;
2014-09-04 00:56:09 +02:00
uintptr n , i , end_b , elemsize , size , ti , objti , count , type , nobj ;
uintptr precise_type , nominal_size ;
const uintptr * pc , * chan_ret ;
uintptr chancap ;
2012-12-22 02:15:33 +01:00
void * obj ;
2014-07-19 10:53:52 +02:00
const Type * t , * et ;
2013-01-29 21:52:43 +01:00
Slice * sliceptr ;
2014-07-19 10:53:52 +02:00
String * stringptr ;
2013-01-29 21:52:43 +01:00
Frame * stack_ptr , stack_top , stack [ GC_STACK_CAPACITY + 4 ] ;
BufferList * scanbuffers ;
2014-06-05 01:15:33 +02:00
Scanbuf sbuf ;
2013-01-29 21:52:43 +01:00
Eface * eface ;
Iface * iface ;
2013-07-16 08:54:42 +02:00
Hchan * chan ;
2014-09-04 00:56:09 +02:00
const ChanType * chantype ;
2014-06-07 00:37:27 +02:00
Obj * wp ;
2012-12-22 02:15:33 +01:00
2014-06-07 00:37:27 +02:00
if ( sizeof ( Workbuf ) % WorkbufSize ! = 0 )
2012-12-22 02:15:33 +01:00
runtime_throw ( " scanblock: size of Workbuf is suboptimal " ) ;
// Memory arena parameters.
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
arena_used = runtime_mheap . arena_used ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
stack_ptr = stack + nelem ( stack ) - 1 ;
2014-06-05 01:15:33 +02:00
2013-01-29 21:52:43 +01:00
precise_type = false ;
nominal_size = 0 ;
2014-06-07 00:37:27 +02:00
if ( wbuf ) {
nobj = wbuf - > nobj ;
wp = & wbuf - > obj [ nobj ] ;
} else {
nobj = 0 ;
wp = nil ;
}
2014-06-05 01:15:33 +02:00
// Initialize sbuf
scanbuffers = & bufferList [ runtime_m ( ) - > helpgc ] ;
sbuf . ptr . begin = sbuf . ptr . pos = & scanbuffers - > ptrtarget [ 0 ] ;
sbuf . ptr . end = sbuf . ptr . begin + nelem ( scanbuffers - > ptrtarget ) ;
sbuf . obj . begin = sbuf . obj . pos = & scanbuffers - > obj [ 0 ] ;
sbuf . obj . end = sbuf . obj . begin + nelem ( scanbuffers - > obj ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
sbuf . wbuf = wbuf ;
sbuf . wp = wp ;
sbuf . nobj = nobj ;
2013-07-16 08:54:42 +02:00
// (Silence the compiler)
chan = nil ;
chantype = nil ;
chan_ret = nil ;
2012-12-22 02:15:33 +01:00
goto next_block ;
for ( ; ; ) {
// Each iteration scans the block b of length n, queueing pointers in
// the work buffer.
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . nbytes , n ) ;
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . obj . sum , sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
runtime_xadd64 ( & gcstats . obj . cnt , 1 ) ;
}
2014-09-04 00:56:09 +02:00
if ( ti ! = 0 ) {
2014-07-19 10:53:52 +02:00
if ( Debug > 1 ) {
runtime_printf ( " scanblock %p %D ti %p \n " , b , ( int64 ) n , ti ) ;
}
2013-01-29 21:52:43 +01:00
pc = ( uintptr * ) ( ti & ~ ( uintptr ) PC_BITS ) ;
precise_type = ( ti & PRECISE ) ;
stack_top . elemsize = pc [ 0 ] ;
if ( ! precise_type )
nominal_size = pc [ 0 ] ;
if ( ti & LOOP ) {
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . loop_or_ret = pc + 1 ;
} else {
stack_top . count = 1 ;
}
2013-07-16 08:54:42 +02:00
if ( Debug ) {
// Simple sanity check for provided type info ti:
// The declared size of the object must be not larger than the actual size
// (it can be smaller due to inferior pointers).
// It's difficult to make a comprehensive check due to inferior pointers,
// reflection, gob, etc.
if ( pc [ 0 ] > n ) {
runtime_printf ( " invalid gc type info: type info size %p, block size %p \n " , pc [ 0 ] , n ) ;
runtime_throw ( " invalid gc type info " ) ;
}
}
2014-09-04 00:56:09 +02:00
} else if ( UseSpanType ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . notype , 1 ) ;
2013-01-29 21:52:43 +01:00
type = runtime_gettype ( b ) ;
if ( type ! = 0 ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . typelookup , 1 ) ;
2013-01-29 21:52:43 +01:00
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
switch ( type & ( PtrSize - 1 ) ) {
case TypeInfo_SingleObject :
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) t - > __gc ;
2013-01-29 21:52:43 +01:00
precise_type = true ; // type information about 'b' is precise
stack_top . count = 1 ;
stack_top . elemsize = pc [ 0 ] ;
break ;
case TypeInfo_Array :
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) t - > __gc ;
2013-01-29 21:52:43 +01:00
if ( pc [ 0 ] = = 0 )
goto next_block ;
precise_type = true ; // type information about 'b' is precise
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . elemsize = pc [ 0 ] ;
stack_top . loop_or_ret = pc + 1 ;
break ;
2013-07-16 08:54:42 +02:00
case TypeInfo_Chan :
chan = ( Hchan * ) b ;
2014-09-04 00:56:09 +02:00
chantype = ( const ChanType * ) t ;
2013-07-16 08:54:42 +02:00
chan_ret = nil ;
pc = chanProg ;
2013-01-29 21:52:43 +01:00
break ;
default :
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D type %p %S \n " , b , ( int64 ) n , type , * t - > string ) ;
2013-01-29 21:52:43 +01:00
runtime_throw ( " scanblock: invalid type " ) ;
return ;
}
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D type %p %S pc=%p \n " , b , ( int64 ) n , type , * t - > string , pc ) ;
2013-01-29 21:52:43 +01:00
} else {
pc = defaultProg ;
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D unknown type \n " , b , ( int64 ) n ) ;
2013-01-29 21:52:43 +01:00
}
} else {
pc = defaultProg ;
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D no span types \n " , b , ( int64 ) n ) ;
2013-01-29 21:52:43 +01:00
}
2012-12-22 02:15:33 +01:00
2013-07-16 08:54:42 +02:00
if ( IgnorePreciseGC )
pc = defaultProg ;
2012-12-22 02:15:33 +01:00
pc + + ;
stack_top . b = ( uintptr ) b ;
end_b = ( uintptr ) b + n - PtrSize ;
2013-01-29 21:52:43 +01:00
for ( ; ; ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . instr [ pc [ 0 ] ] , 1 ) ;
2013-01-29 21:52:43 +01:00
obj = nil ;
objti = 0 ;
2012-12-22 02:15:33 +01:00
switch ( pc [ 0 ] ) {
2013-01-29 21:52:43 +01:00
case GC_PTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
objti = pc [ 2 ] ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_ptr @%p: %p ti=%p \n " , stack_top . b + pc [ 1 ] , obj , objti ) ;
2013-01-29 21:52:43 +01:00
pc + = 3 ;
2013-07-16 08:54:42 +02:00
if ( Debug )
checkptr ( obj , objti ) ;
2013-01-29 21:52:43 +01:00
break ;
case GC_SLICE :
sliceptr = ( Slice * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_slice @%p: %p/%D/%D \n " , sliceptr , sliceptr - > array , ( int64 ) sliceptr - > __count , ( int64 ) sliceptr - > cap ) ;
2013-01-29 21:52:43 +01:00
if ( sliceptr - > cap ! = 0 ) {
obj = sliceptr - > array ;
2013-07-16 08:54:42 +02:00
// Can't use slice element type for scanning,
// because if it points to an array embedded
// in the beginning of a struct,
// we will scan the whole struct as the slice.
// So just obtain type info from heap.
2013-01-29 21:52:43 +01:00
}
pc + = 3 ;
break ;
case GC_APTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_aptr @%p: %p \n " , stack_top . b + pc [ 1 ] , obj ) ;
2013-01-29 21:52:43 +01:00
pc + = 2 ;
break ;
case GC_STRING :
2014-07-19 10:53:52 +02:00
stringptr = ( String * ) ( stack_top . b + pc [ 1 ] ) ;
if ( Debug > 2 )
runtime_printf ( " gc_string @%p: %p/%D \n " , stack_top . b + pc [ 1 ] , stringptr - > str , ( int64 ) stringptr - > len ) ;
if ( stringptr - > len ! = 0 )
markonly ( stringptr - > str ) ;
2013-01-29 21:52:43 +01:00
pc + = 2 ;
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
case GC_EFACE :
eface = ( Eface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_eface @%p: %p %p \n " , stack_top . b + pc [ 1 ] , eface - > __type_descriptor , eface - > __object ) ;
2014-06-07 00:37:27 +02:00
if ( eface - > __type_descriptor = = nil )
2013-07-16 08:54:42 +02:00
continue ;
// eface->type
2014-06-07 00:37:27 +02:00
t = eface - > __type_descriptor ;
2013-07-16 08:54:42 +02:00
if ( ( const byte * ) t > = arena_start & & ( const byte * ) t < arena_used ) {
union { const Type * tc ; Type * tr ; } u ;
u . tc = t ;
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { u . tr , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
// eface->__object
if ( ( byte * ) eface - > __object > = arena_start & & ( byte * ) eface - > __object < arena_used ) {
2014-09-04 00:56:09 +02:00
if ( __go_is_pointer_type ( t ) ) {
2014-06-07 00:37:27 +02:00
if ( ( t - > __code & KindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = eface - > __object ;
2015-01-15 01:27:56 +01:00
if ( ( t - > __code & kindMask ) = = KindPtr ) {
2014-07-19 10:53:52 +02:00
// Only use type information if it is a pointer-containing type.
// This matches the GC programs written by cmd/gc/reflect.c's
// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
et = ( ( const PtrType * ) t ) - > elem ;
if ( ! ( et - > __code & KindNoPointers ) )
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) ( ( const PtrType * ) t ) - > elem - > __gc ;
2014-07-19 10:53:52 +02:00
}
2013-01-29 21:52:43 +01:00
} else {
obj = eface - > __object ;
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) t - > __gc ;
2013-01-29 21:52:43 +01:00
}
}
break ;
case GC_IFACE :
iface = ( Iface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_iface @%p: %p/%p %p \n " , stack_top . b + pc [ 1 ] , iface - > __methods [ 0 ] , nil , iface - > __object ) ;
2013-01-29 21:52:43 +01:00
if ( iface - > tab = = nil )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
// iface->tab
if ( ( byte * ) iface - > tab > = arena_start & & ( byte * ) iface - > tab < arena_used ) {
2014-09-04 00:56:09 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { iface - > tab , 0 } ;
2014-06-05 01:15:33 +02:00
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
// iface->data
if ( ( byte * ) iface - > __object > = arena_start & & ( byte * ) iface - > __object < arena_used ) {
2014-09-04 00:56:09 +02:00
t = ( const Type * ) iface - > tab [ 0 ] ;
if ( __go_is_pointer_type ( t ) ) {
2014-06-07 00:37:27 +02:00
if ( ( t - > __code & KindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = iface - > __object ;
2015-01-15 01:27:56 +01:00
if ( ( t - > __code & kindMask ) = = KindPtr ) {
2014-07-19 10:53:52 +02:00
// Only use type information if it is a pointer-containing type.
// This matches the GC programs written by cmd/gc/reflect.c's
// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
et = ( ( const PtrType * ) t ) - > elem ;
if ( ! ( et - > __code & KindNoPointers ) )
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) ( ( const PtrType * ) t ) - > elem - > __gc ;
2014-07-19 10:53:52 +02:00
}
2013-01-29 21:52:43 +01:00
} else {
obj = iface - > __object ;
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) t - > __gc ;
2013-01-29 21:52:43 +01:00
}
}
break ;
2012-12-22 02:15:33 +01:00
case GC_DEFAULT_PTR :
2013-07-16 08:54:42 +02:00
while ( stack_top . b < = end_b ) {
obj = * ( byte * * ) stack_top . b ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_default_ptr @%p: %p \n " , stack_top . b , obj ) ;
2012-12-22 02:15:33 +01:00
stack_top . b + = PtrSize ;
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
}
}
2013-01-29 21:52:43 +01:00
goto next_block ;
case GC_END :
if ( - - stack_top . count ! = 0 ) {
// Next iteration of a loop if possible.
2013-07-16 08:54:42 +02:00
stack_top . b + = stack_top . elemsize ;
if ( stack_top . b + stack_top . elemsize < = end_b + PtrSize ) {
2013-01-29 21:52:43 +01:00
pc = stack_top . loop_or_ret ;
continue ;
}
i = stack_top . b ;
} else {
// Stack pop if possible.
if ( stack_ptr + 1 < stack + nelem ( stack ) ) {
pc = stack_top . loop_or_ret ;
stack_top = * ( + + stack_ptr ) ;
continue ;
}
i = ( uintptr ) b + nominal_size ;
}
if ( ! precise_type ) {
// Quickly scan [b+i,b+n) for possible pointers.
for ( ; i < = end_b ; i + = PtrSize ) {
if ( * ( byte * * ) i ! = nil ) {
// Found a value that may be a pointer.
// Do a rescan of the entire block.
2014-06-05 01:15:33 +02:00
enqueue ( ( Obj ) { b , n , 0 } , & sbuf . wbuf , & sbuf . wp , & sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . rescan , 1 ) ;
runtime_xadd64 ( & gcstats . rescanbytes , n ) ;
}
2013-01-29 21:52:43 +01:00
break ;
}
}
}
goto next_block ;
case GC_ARRAY_START :
i = stack_top . b + pc [ 1 ] ;
count = pc [ 2 ] ;
elemsize = pc [ 3 ] ;
pc + = 4 ;
// Stack push.
* stack_ptr - - = stack_top ;
stack_top = ( Frame ) { count , elemsize , i , pc } ;
continue ;
case GC_ARRAY_NEXT :
if ( - - stack_top . count ! = 0 ) {
stack_top . b + = stack_top . elemsize ;
pc = stack_top . loop_or_ret ;
} else {
// Stack pop.
stack_top = * ( + + stack_ptr ) ;
pc + = 1 ;
}
continue ;
case GC_CALL :
// Stack push.
* stack_ptr - - = stack_top ;
stack_top = ( Frame ) { 1 , 0 , stack_top . b + pc [ 1 ] , pc + 3 /*return address*/ } ;
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) ( ( const byte * ) pc + * ( const int32 * ) ( pc + 2 ) ) ; // target of the CALL instruction
2013-01-29 21:52:43 +01:00
continue ;
case GC_REGION :
obj = ( void * ) ( stack_top . b + pc [ 1 ] ) ;
2013-07-16 08:54:42 +02:00
size = pc [ 2 ] ;
objti = pc [ 3 ] ;
2013-01-29 21:52:43 +01:00
pc + = 4 ;
2013-07-16 08:54:42 +02:00
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_region @%p: %D %p \n " , stack_top . b + pc [ 1 ] , ( int64 ) size , objti ) ;
2014-06-05 01:15:33 +02:00
* sbuf . obj . pos + + = ( Obj ) { obj , size , objti } ;
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
continue ;
case GC_CHAN_PTR :
chan = * ( Hchan * * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 & & chan ! = nil )
runtime_printf ( " gc_chan_ptr @%p: %p/%D/%D %p \n " , stack_top . b + pc [ 1 ] , chan , ( int64 ) chan - > qcount , ( int64 ) chan - > dataqsiz , pc [ 2 ] ) ;
2013-07-16 08:54:42 +02:00
if ( chan = = nil ) {
pc + = 3 ;
continue ;
}
if ( markonly ( chan ) ) {
chantype = ( ChanType * ) pc [ 2 ] ;
2014-06-07 00:37:27 +02:00
if ( ! ( chantype - > elem - > __code & KindNoPointers ) ) {
2013-07-16 08:54:42 +02:00
// Start chanProg.
chan_ret = pc + 3 ;
pc = chanProg + 1 ;
continue ;
}
}
pc + = 3 ;
continue ;
case GC_CHAN :
// There are no heap pointers in struct Hchan,
// so we can ignore the leading sizeof(Hchan) bytes.
2014-06-07 00:37:27 +02:00
if ( ! ( chantype - > elem - > __code & KindNoPointers ) ) {
2013-07-16 08:54:42 +02:00
// Channel's buffer follows Hchan immediately in memory.
// Size of buffer (cap(c)) is second int in the chan struct.
chancap = ( ( uintgo * ) chan ) [ 1 ] ;
if ( chancap > 0 ) {
// TODO(atom): split into two chunks so that only the
// in-use part of the circular buffer is scanned.
// (Channel routines zero the unused part, so the current
// code does not lead to leaks, it's just a little inefficient.)
2014-09-04 00:56:09 +02:00
* sbuf . obj . pos + + = ( Obj ) { ( byte * ) chan + runtime_Hchansize , chancap * chantype - > elem - > __size ,
( uintptr ) chantype - > elem - > __gc | PRECISE | LOOP } ;
2014-06-05 01:15:33 +02:00
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
}
if ( chan_ret = = nil )
goto next_block ;
pc = chan_ret ;
continue ;
2012-12-22 02:15:33 +01:00
default :
2014-07-12 02:01:09 +02:00
runtime_printf ( " runtime: invalid GC instruction %p at %p \n " , pc [ 0 ] , pc ) ;
2012-12-22 02:15:33 +01:00
runtime_throw ( " scanblock: invalid GC instruction " ) ;
return ;
}
2013-01-29 21:52:43 +01:00
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , objti } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
}
2012-12-22 02:15:33 +01:00
next_block :
2011-03-17 00:05:44 +01:00
// Done scanning [b, b+n). Prepare for the next iteration of
2013-01-29 21:52:43 +01:00
// the loop by setting b, n, ti to the parameters for the next block.
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
flushptrbuf ( & sbuf ) ;
flushobjbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
2012-12-22 02:15:33 +01:00
if ( ! keepworking ) {
2014-06-05 01:15:33 +02:00
if ( sbuf . wbuf )
putempty ( sbuf . wbuf ) ;
return ;
2012-12-22 02:15:33 +01:00
}
// Emptied our buffer: refill.
2014-06-05 01:15:33 +02:00
sbuf . wbuf = getfull ( sbuf . wbuf ) ;
if ( sbuf . wbuf = = nil )
return ;
sbuf . nobj = sbuf . wbuf - > nobj ;
sbuf . wp = sbuf . wbuf - > obj + sbuf . wbuf - > nobj ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
2012-12-22 02:15:33 +01:00
// Fetch b from the work buffer.
2014-06-05 01:15:33 +02:00
- - sbuf . wp ;
b = sbuf . wp - > p ;
n = sbuf . wp - > n ;
ti = sbuf . wp - > ti ;
sbuf . nobj - - ;
2011-03-17 00:05:44 +01:00
}
}
2014-06-07 00:37:27 +02:00
static struct root_list * roots ;
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
void
__go_register_gc_roots ( struct root_list * r )
{
// FIXME: This needs locking if multiple goroutines can call
// dlopen simultaneously.
r - > next = roots ;
roots = r ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
// Append obj to the work buffer.
// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
static void
enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj )
{
uintptr nobj , off ;
Obj * wp ;
Workbuf * wbuf ;
if ( Debug > 1 )
runtime_printf ( " append obj(%p %D %p) \n " , obj . p , ( int64 ) obj . n , obj . ti ) ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
return ;
// Load work buffer state
wp = * _wp ;
wbuf = * _wbuf ;
nobj = * _nobj ;
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
// Save work buffer state
* _wp = wp ;
* _wbuf = wbuf ;
* _nobj = nobj ;
}
2014-06-07 00:37:27 +02:00
static void
enqueue1 ( Workbuf * * wbufp , Obj obj )
{
Workbuf * wbuf ;
wbuf = * wbufp ;
if ( wbuf - > nobj > = nelem ( wbuf - > obj ) )
* wbufp = wbuf = getempty ( wbuf ) ;
wbuf - > obj [ wbuf - > nobj + + ] = obj ;
}
2012-10-23 06:31:11 +02:00
static void
markroot ( ParFor * desc , uint32 i )
{
2012-12-22 02:15:33 +01:00
Workbuf * wbuf ;
2014-06-07 00:37:27 +02:00
FinBlock * fb ;
MHeap * h ;
MSpan * * allspans , * s ;
uint32 spanidx , sg ;
G * gp ;
void * p ;
2012-12-22 02:15:33 +01:00
2012-10-23 06:31:11 +02:00
USED ( & desc ) ;
2014-06-07 00:37:27 +02:00
wbuf = getempty ( nil ) ;
2014-07-19 10:53:52 +02:00
// Note: if you add a case here, please also update heapdump.c:dumproots.
2014-06-07 00:37:27 +02:00
switch ( i ) {
case RootData :
// For gccgo this is both data and bss.
{
struct root_list * pl ;
for ( pl = roots ; pl ! = nil ; pl = pl - > next ) {
struct root * pr = & pl - > roots [ 0 ] ;
while ( 1 ) {
void * decl = pr - > decl ;
if ( decl = = nil )
break ;
enqueue1 ( & wbuf , ( Obj ) { decl , pr - > size , 0 } ) ;
pr + + ;
}
}
}
break ;
case RootBss :
// For gccgo we use this for all the other global roots.
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_m0 , sizeof runtime_m0 , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_g0 , sizeof runtime_g0 , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allg , sizeof runtime_allg , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allm , sizeof runtime_allm , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allp , sizeof runtime_allp , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & work , sizeof work , 0 } ) ;
runtime_proc_scan ( & wbuf , enqueue1 ) ;
runtime_MProf_Mark ( & wbuf , enqueue1 ) ;
runtime_time_scan ( & wbuf , enqueue1 ) ;
runtime_netpoll_scan ( & wbuf , enqueue1 ) ;
break ;
case RootFinalizers :
for ( fb = allfin ; fb ; fb = fb - > alllink )
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) fb - > fin , fb - > cnt * sizeof ( fb - > fin [ 0 ] ) , 0 } ) ;
break ;
case RootSpanTypes :
// mark span types and MSpan.specials (to walk spans only once)
h = & runtime_mheap ;
sg = h - > sweepgen ;
allspans = h - > allspans ;
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
Special * sp ;
SpecialFinalizer * spf ;
s = allspans [ spanidx ] ;
if ( s - > sweepgen ! = sg ) {
runtime_printf ( " sweep %d %d \n " , s - > sweepgen , sg ) ;
runtime_throw ( " gc: unswept span " ) ;
}
if ( s - > state ! = MSpanInUse )
continue ;
// The garbage collector ignores type pointers stored in MSpan.types:
// - Compiler-generated types are stored outside of heap.
// - The reflect package has runtime-generated types cached in its data structures.
// The garbage collector relies on finding the references via that cache.
if ( s - > types . compression = = MTypes_Words | | s - > types . compression = = MTypes_Bytes )
markonly ( ( byte * ) s - > types . data ) ;
for ( sp = s - > specials ; sp ! = nil ; sp = sp - > next ) {
if ( sp - > kind ! = KindSpecialFinalizer )
continue ;
// don't mark finalized object, but scan it so we
// retain everything it points to.
spf = ( SpecialFinalizer * ) sp ;
// A finalizer can be set for an inner byte of an object, find object beginning.
p = ( void * ) ( ( s - > start < < PageShift ) + spf - > offset / s - > elemsize * s - > elemsize ) ;
enqueue1 ( & wbuf , ( Obj ) { p , s - > elemsize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > fn , PtrSize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > ft , PtrSize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > ot , PtrSize , 0 } ) ;
}
}
break ;
case RootFlushCaches :
flushallmcaches ( ) ;
break ;
default :
// the rest is scanning goroutine stacks
if ( i - RootCount > = runtime_allglen )
runtime_throw ( " markroot: bad index " ) ;
gp = runtime_allg [ i - RootCount ] ;
// remember when we've first observed the G blocked
// needed only to output in traceback
if ( ( gp - > status = = Gwaiting | | gp - > status = = Gsyscall ) & & gp - > waitsince = = 0 )
gp - > waitsince = work . tstart ;
addstackroots ( gp , & wbuf ) ;
break ;
}
if ( wbuf )
scanblock ( wbuf , false ) ;
2012-10-23 06:31:11 +02:00
}
2015-10-31 01:59:47 +01:00
static const FuncVal markroot_funcval = { ( void * ) markroot } ;
2011-03-17 00:05:44 +01:00
// Get an empty work buffer off the work.empty list,
// allocating new buffers as needed.
static Workbuf *
getempty ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
runtime_lfstackpush ( & work . full , & b - > node ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . empty ) ;
if ( b = = nil ) {
// Need to allocate.
runtime_lock ( & work ) ;
if ( work . nchunk < sizeof * b ) {
work . nchunk = 1 < < 20 ;
2013-11-06 20:49:01 +01:00
work . chunk = runtime_SysAlloc ( work . nchunk , & mstats . gc_sys ) ;
2013-07-16 08:54:42 +02:00
if ( work . chunk = = nil )
runtime_throw ( " runtime: cannot allocate memory " ) ;
2011-10-27 01:57:58 +02:00
}
2012-10-23 06:31:11 +02:00
b = ( Workbuf * ) work . chunk ;
work . chunk + = sizeof * b ;
work . nchunk - = sizeof * b ;
runtime_unlock ( & work ) ;
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
b - > nobj = 0 ;
2011-03-17 00:05:44 +01:00
return b ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
static void
putempty ( Workbuf * b )
{
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . putempty , 1 ) ;
2012-10-23 06:31:11 +02:00
runtime_lfstackpush ( & work . empty , & b - > node ) ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
// Get a full work buffer off the work.full list, or return nil.
static Workbuf *
getfull ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 i ;
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . getfull , 1 ) ;
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
runtime_lfstackpush ( & work . empty , & b - > node ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil | | work . nproc = = 1 )
2011-10-27 01:57:58 +02:00
return b ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
runtime_xadd ( & work . nwait , + 1 ) ;
for ( i = 0 ; ; i + + ) {
2012-10-23 06:31:11 +02:00
if ( work . full ! = 0 ) {
runtime_xadd ( & work . nwait , - 1 ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil )
return b ;
runtime_xadd ( & work . nwait , + 1 ) ;
2011-10-27 01:57:58 +02:00
}
if ( work . nwait = = work . nproc )
return nil ;
2012-10-23 06:31:11 +02:00
if ( i < 10 ) {
m - > gcstats . nprocyield + + ;
2011-10-27 01:57:58 +02:00
runtime_procyield ( 20 ) ;
2012-10-23 06:31:11 +02:00
} else if ( i < 20 ) {
m - > gcstats . nosyield + + ;
2011-10-27 01:57:58 +02:00
runtime_osyield ( ) ;
2012-10-23 06:31:11 +02:00
} else {
m - > gcstats . nsleep + + ;
2011-10-27 01:57:58 +02:00
runtime_usleep ( 100 ) ;
2012-10-23 06:31:11 +02:00
}
2011-03-17 00:05:44 +01:00
}
}
2011-10-27 01:57:58 +02:00
static Workbuf *
handoff ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 n ;
Workbuf * b1 ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
// Make new buffer with half of b's pointers.
b1 = getempty ( nil ) ;
n = b - > nobj / 2 ;
b - > nobj - = n ;
b1 - > nobj = n ;
runtime_memmove ( b1 - > obj , b - > obj + b - > nobj , n * sizeof b1 - > obj [ 0 ] ) ;
2012-10-23 06:31:11 +02:00
m - > gcstats . nhandoff + + ;
m - > gcstats . nhandoffcnt + = n ;
2011-10-27 01:57:58 +02:00
// Put b on full list - let first half of b get stolen.
2012-10-23 06:31:11 +02:00
runtime_lfstackpush ( & work . full , & b - > node ) ;
2011-10-27 01:57:58 +02:00
return b1 ;
}
2011-11-28 06:45:49 +01:00
static void
2014-06-07 00:37:27 +02:00
addstackroots ( G * gp , Workbuf * * wbufp )
2012-10-23 06:31:11 +02:00
{
2014-06-07 00:37:27 +02:00
switch ( gp - > status ) {
default :
runtime_printf ( " unexpected G.status %d (goroutine %p %D) \n " , gp - > status , gp , gp - > goid ) ;
runtime_throw ( " mark - bad status " ) ;
case Gdead :
return ;
case Grunning :
runtime_throw ( " mark - world not stopped " ) ;
case Grunnable :
case Gsyscall :
case Gwaiting :
break ;
2012-10-23 06:31:11 +02:00
}
2011-11-28 06:45:49 +01:00
# ifdef USING_SPLIT_STACK
M * mp ;
void * sp ;
size_t spsize ;
void * next_segment ;
void * next_sp ;
void * initial_sp ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
sp = __splitstack_find ( nil , nil , & spsize , & next_segment ,
& next_sp , & initial_sp ) ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
// The exception is that if the goroutine is about to enter or might
// have just exited a system call, it may be executing code such
// as schedlock and may have needed to start a new stack segment.
// Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot.
if ( gp - > gcstack ! = nil ) {
sp = gp - > gcstack ;
spsize = gp - > gcstack_size ;
next_segment = gp - > gcnext_segment ;
next_sp = gp - > gcnext_sp ;
initial_sp = gp - > gcinitial_sp ;
} else {
sp = __splitstack_find_context ( & gp - > stack_context [ 0 ] ,
& spsize , & next_segment ,
& next_sp , & initial_sp ) ;
}
}
if ( sp ! = nil ) {
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
while ( ( sp = __splitstack_find ( next_segment , next_sp ,
& spsize , & next_segment ,
& next_sp , & initial_sp ) ) ! = nil )
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
}
# else
M * mp ;
byte * bottom ;
byte * top ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
bottom = ( byte * ) & gp ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
bottom = ( byte * ) gp - > gcnext_sp ;
if ( bottom = = nil )
return ;
}
top = ( byte * ) gp - > gcinitial_sp + gp - > gcstack_size ;
if ( top > bottom )
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { bottom , top - bottom , 0 } ) ;
2011-11-28 06:45:49 +01:00
else
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { top , bottom - top , 0 } ) ;
2011-11-28 06:45:49 +01:00
# endif
}
2010-12-03 05:34:57 +01:00
void
2014-06-07 00:37:27 +02:00
runtime_queuefinalizer ( void * p , FuncVal * fn , const FuncType * ft , const PtrType * ot )
2011-10-27 01:57:58 +02:00
{
FinBlock * block ;
Finalizer * f ;
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
if ( finq = = nil | | finq - > cnt = = finq - > cap ) {
if ( finc = = nil ) {
2014-06-07 00:37:27 +02:00
finc = runtime_persistentalloc ( FinBlockSize , 0 , & mstats . gc_sys ) ;
finc - > cap = ( FinBlockSize - sizeof ( FinBlock ) ) / sizeof ( Finalizer ) + 1 ;
2011-10-27 01:57:58 +02:00
finc - > alllink = allfin ;
allfin = finc ;
}
block = finc ;
finc = block - > next ;
block - > next = finq ;
finq = block ;
}
f = & finq - > fin [ finq - > cnt ] ;
finq - > cnt + + ;
f - > fn = fn ;
f - > ft = ft ;
2013-11-06 20:49:01 +01:00
f - > ot = ot ;
2011-10-27 01:57:58 +02:00
f - > arg = p ;
2014-07-19 10:53:52 +02:00
runtime_fingwake = true ;
runtime_unlock ( & finlock ) ;
}
void
runtime_iterate_finq ( void ( * callback ) ( FuncVal * , void * , const FuncType * , const PtrType * ) )
{
FinBlock * fb ;
Finalizer * f ;
int32 i ;
for ( fb = allfin ; fb ; fb = fb - > alllink ) {
for ( i = 0 ; i < fb - > cnt ; i + + ) {
f = & fb - > fin [ i ] ;
callback ( f - > fn , f - > arg , f - > ft , f - > ot ) ;
}
}
2014-06-07 00:37:27 +02:00
}
void
runtime_MSpan_EnsureSwept ( MSpan * s )
{
M * m = runtime_m ( ) ;
2014-07-19 10:53:52 +02:00
G * g = runtime_g ( ) ;
2014-06-07 00:37:27 +02:00
uint32 sg ;
2014-07-19 10:53:52 +02:00
// Caller must disable preemption.
// Otherwise when this function returns the span can become unswept again
// (if GC is triggered on another goroutine).
if ( m - > locks = = 0 & & m - > mallocing = = 0 & & g ! = m - > g0 )
runtime_throw ( " MSpan_EnsureSwept: m is not locked " ) ;
2014-06-07 00:37:27 +02:00
sg = runtime_mheap . sweepgen ;
if ( runtime_atomicload ( & s - > sweepgen ) = = sg )
return ;
if ( runtime_cas ( & s - > sweepgen , sg - 2 , sg - 1 ) ) {
runtime_MSpan_Sweep ( s ) ;
return ;
}
// unfortunate condition, and we don't have efficient means to wait
while ( runtime_atomicload ( & s - > sweepgen ) ! = sg )
runtime_osyield ( ) ;
2011-10-27 01:57:58 +02:00
}
// Sweep frees or collects finalizers for blocks not marked in the mark phase.
2011-03-17 00:05:44 +01:00
// It clears the mark bits in preparation for the next GC round.
2014-06-07 00:37:27 +02:00
// Returns true if the span was returned to heap.
bool
runtime_MSpan_Sweep ( MSpan * s )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2014-06-07 00:37:27 +02:00
int32 cl , n , npages , nfree ;
uintptr size , off , * bitp , shift , bits ;
uint32 sweepgen ;
2010-12-03 05:34:57 +01:00
byte * p ;
MCache * c ;
2011-10-27 01:57:58 +02:00
byte * arena_start ;
2012-10-23 06:31:11 +02:00
MLink head , * end ;
byte * type_data ;
byte compression ;
uintptr type_data_inc ;
2014-06-05 01:15:33 +02:00
MLink * x ;
2014-06-07 00:37:27 +02:00
Special * special , * * specialp , * y ;
bool res , sweepgenset ;
2011-10-27 01:57:58 +02:00
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2012-10-23 06:31:11 +02:00
2014-06-07 00:37:27 +02:00
// It's critical that we enter this function with preemption disabled,
// GC must not start while we are in the middle of this function.
if ( m - > locks = = 0 & & m - > mallocing = = 0 & & runtime_g ( ) ! = m - > g0 )
runtime_throw ( " MSpan_Sweep: m is not locked " ) ;
sweepgen = runtime_mheap . sweepgen ;
if ( s - > state ! = MSpanInUse | | s - > sweepgen ! = sweepgen - 1 ) {
runtime_printf ( " MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d \n " ,
s - > state , s - > sweepgen , sweepgen ) ;
runtime_throw ( " MSpan_Sweep: bad span state " ) ;
}
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-10-23 06:31:11 +02:00
cl = s - > sizeclass ;
size = s - > elemsize ;
if ( cl = = 0 ) {
n = 1 ;
} else {
// Chunk full of small blocks.
npages = runtime_class_to_allocnpages [ cl ] ;
n = ( npages < < PageShift ) / size ;
}
2014-06-07 00:37:27 +02:00
res = false ;
2012-10-23 06:31:11 +02:00
nfree = 0 ;
end = & head ;
c = m - > mcache ;
2014-06-07 00:37:27 +02:00
sweepgenset = false ;
2014-06-05 01:15:33 +02:00
// mark any free objects in this span so we don't collect them
for ( x = s - > freelist ; x ! = nil ; x = x - > next ) {
// This is markonly(x) but faster because we don't need
// atomic access and we're guaranteed to be pointing at
// the head of a valid object.
off = ( uintptr * ) x - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
* bitp | = bitMarked < < shift ;
}
2014-07-19 10:53:52 +02:00
2014-06-07 00:37:27 +02:00
// Unlink & free special records for any objects we're about to free.
specialp = & s - > specials ;
special = * specialp ;
while ( special ! = nil ) {
// A finalizer can be set for an inner byte of an object, find object beginning.
p = ( byte * ) ( s - > start < < PageShift ) + special - > offset / size * size ;
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
if ( ( bits & ( bitAllocated | bitMarked ) ) = = bitAllocated ) {
// Find the exact byte for which the special was setup
// (as opposed to object beginning).
p = ( byte * ) ( s - > start < < PageShift ) + special - > offset ;
// about to free object: splice out special record
y = special ;
special = special - > next ;
* specialp = special ;
if ( ! runtime_freespecial ( y , p , size , false ) ) {
// stop freeing of object if it has a finalizer
* bitp | = bitMarked < < shift ;
}
} else {
// object is still live: keep special record
specialp = & special - > next ;
special = * specialp ;
}
}
2012-10-23 06:31:11 +02:00
type_data = ( byte * ) s - > types . data ;
type_data_inc = sizeof ( uintptr ) ;
compression = s - > types . compression ;
switch ( compression ) {
case MTypes_Bytes :
type_data + = 8 * sizeof ( uintptr ) ;
type_data_inc = 1 ;
break ;
}
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
// Sweep through n objects of given size starting at p.
// This thread owns the span now, so it can manipulate
// the block bitmap without atomic operations.
2014-06-07 00:37:27 +02:00
p = ( byte * ) ( s - > start < < PageShift ) ;
2012-10-23 06:31:11 +02:00
for ( ; n > 0 ; n - - , p + = size , type_data + = type_data_inc ) {
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
2012-03-02 21:01:37 +01:00
2012-10-23 06:31:11 +02:00
if ( ( bits & bitAllocated ) = = 0 )
2011-03-17 00:05:44 +01:00
continue ;
2012-10-23 06:31:11 +02:00
if ( ( bits & bitMarked ) ! = 0 ) {
* bitp & = ~ ( bitMarked < < shift ) ;
continue ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
if ( runtime_debug . allocfreetrace )
runtime_tracefree ( p , size ) ;
// Clear mark and scan bits.
* bitp & = ~ ( ( bitScan | bitMarked ) < < shift ) ;
2011-03-17 00:05:44 +01:00
2012-10-23 06:31:11 +02:00
if ( cl = = 0 ) {
// Free large span.
runtime_unmarkspan ( p , 1 < < PageShift ) ;
2014-06-07 00:37:27 +02:00
s - > needzero = 1 ;
// important to set sweepgen before returning it to heap
runtime_atomicstore ( & s - > sweepgen , sweepgen ) ;
sweepgenset = true ;
2014-07-19 10:53:52 +02:00
// See note about SysFault vs SysFree in malloc.goc.
2014-06-05 01:15:33 +02:00
if ( runtime_debug . efence )
2014-07-19 10:53:52 +02:00
runtime_SysFault ( p , size ) ;
2014-06-05 01:15:33 +02:00
else
runtime_MHeap_Free ( & runtime_mheap , s , 1 ) ;
2013-11-06 20:49:01 +01:00
c - > local_nlargefree + + ;
c - > local_largefree + = size ;
2014-06-07 00:37:27 +02:00
runtime_xadd64 ( & mstats . next_gc , - ( uint64 ) ( size * ( gcpercent + 100 ) / 100 ) ) ;
res = true ;
2012-10-23 06:31:11 +02:00
} else {
// Free small object.
switch ( compression ) {
case MTypes_Words :
* ( uintptr * ) type_data = 0 ;
break ;
case MTypes_Bytes :
* ( byte * ) type_data = 0 ;
break ;
}
2014-06-07 00:37:27 +02:00
if ( size > 2 * sizeof ( uintptr ) )
2013-07-16 08:54:42 +02:00
( ( uintptr * ) p ) [ 1 ] = ( uintptr ) 0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed"
2014-06-07 00:37:27 +02:00
else if ( size > sizeof ( uintptr ) )
( ( uintptr * ) p ) [ 1 ] = 0 ;
2012-10-23 06:31:11 +02:00
end - > next = ( MLink * ) p ;
end = ( MLink * ) p ;
nfree + + ;
2010-12-03 05:34:57 +01:00
}
}
2012-10-23 06:31:11 +02:00
2014-07-19 10:53:52 +02:00
// We need to set s->sweepgen = h->sweepgen only when all blocks are swept,
// because of the potential for a concurrent free/SetFinalizer.
// But we need to set it before we make the span available for allocation
// (return it to heap or mcentral), because allocation code assumes that a
// span is already swept if available for allocation.
if ( ! sweepgenset & & nfree = = 0 ) {
2014-06-07 00:37:27 +02:00
// The span must be in our exclusive ownership until we update sweepgen,
// check for potential races.
if ( s - > state ! = MSpanInUse | | s - > sweepgen ! = sweepgen - 1 ) {
runtime_printf ( " MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d \n " ,
s - > state , s - > sweepgen , sweepgen ) ;
runtime_throw ( " MSpan_Sweep: bad span state after sweep " ) ;
}
runtime_atomicstore ( & s - > sweepgen , sweepgen ) ;
}
2014-07-19 10:53:52 +02:00
if ( nfree > 0 ) {
2013-11-06 20:49:01 +01:00
c - > local_nsmallfree [ cl ] + = nfree ;
2012-10-23 06:31:11 +02:00
c - > local_cachealloc - = nfree * size ;
2014-06-07 00:37:27 +02:00
runtime_xadd64 ( & mstats . next_gc , - ( uint64 ) ( nfree * size * ( gcpercent + 100 ) / 100 ) ) ;
res = runtime_MCentral_FreeSpan ( & runtime_mheap . central [ cl ] , s , nfree , head . next , end ) ;
2014-07-19 10:53:52 +02:00
//MCentral_FreeSpan updates sweepgen
2014-06-07 00:37:27 +02:00
}
return res ;
}
// State of background sweep.
2014-09-04 00:56:09 +02:00
// Protected by gclock.
2014-06-07 00:37:27 +02:00
static struct
{
G * g ;
bool parked ;
MSpan * * spans ;
uint32 nspan ;
uint32 spanidx ;
} sweep ;
// background sweeping goroutine
static void
bgsweep ( void * dummy __attribute__ ( ( unused ) ) )
{
runtime_g ( ) - > issystem = 1 ;
for ( ; ; ) {
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 ) {
gcstats . nbgsweep + + ;
runtime_gosched ( ) ;
}
runtime_lock ( & gclock ) ;
2014-07-19 10:53:52 +02:00
if ( ! runtime_mheap . sweepdone ) {
// It's possible if GC has happened between sweepone has
// returned -1 and gclock lock.
runtime_unlock ( & gclock ) ;
continue ;
2014-06-07 00:37:27 +02:00
}
sweep . parked = true ;
2014-07-19 10:53:52 +02:00
runtime_g ( ) - > isbackground = true ;
2014-06-07 00:37:27 +02:00
runtime_parkunlock ( & gclock , " GC sweep wait " ) ;
2014-07-19 10:53:52 +02:00
runtime_g ( ) - > isbackground = false ;
2014-06-07 00:37:27 +02:00
}
}
// sweeps one span
// returns number of pages returned to heap, or -1 if there is nothing to sweep
uintptr
runtime_sweepone ( void )
{
M * m = runtime_m ( ) ;
MSpan * s ;
uint32 idx , sg ;
uintptr npages ;
// increment locks to ensure that the goroutine is not preempted
// in the middle of sweep thus leaving the span in an inconsistent state for next GC
m - > locks + + ;
sg = runtime_mheap . sweepgen ;
for ( ; ; ) {
idx = runtime_xadd ( & sweep . spanidx , 1 ) - 1 ;
if ( idx > = sweep . nspan ) {
runtime_mheap . sweepdone = true ;
m - > locks - - ;
return ( uintptr ) - 1 ;
}
s = sweep . spans [ idx ] ;
if ( s - > state ! = MSpanInUse ) {
s - > sweepgen = sg ;
continue ;
}
if ( s - > sweepgen ! = sg - 2 | | ! runtime_cas ( & s - > sweepgen , sg - 2 , sg - 1 ) )
continue ;
2014-07-19 10:53:52 +02:00
if ( s - > incache )
runtime_throw ( " sweep of incache span " ) ;
2014-06-07 00:37:27 +02:00
npages = s - > npages ;
if ( ! runtime_MSpan_Sweep ( s ) )
npages = 0 ;
m - > locks - - ;
return npages ;
2012-10-23 06:31:11 +02:00
}
2010-12-03 05:34:57 +01:00
}
2012-11-21 08:03:38 +01:00
static void
dumpspan ( uint32 idx )
{
int32 sizeclass , n , npages , i , column ;
uintptr size ;
byte * p ;
byte * arena_start ;
MSpan * s ;
2014-07-19 10:53:52 +02:00
bool allocated ;
2012-11-21 08:03:38 +01:00
2013-11-06 20:49:01 +01:00
s = runtime_mheap . allspans [ idx ] ;
2012-11-21 08:03:38 +01:00
if ( s - > state ! = MSpanInUse )
return ;
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-11-21 08:03:38 +01:00
p = ( byte * ) ( s - > start < < PageShift ) ;
sizeclass = s - > sizeclass ;
size = s - > elemsize ;
if ( sizeclass = = 0 ) {
n = 1 ;
} else {
npages = runtime_class_to_allocnpages [ sizeclass ] ;
n = ( npages < < PageShift ) / size ;
}
runtime_printf ( " %p .. %p: \n " , p , p + n * size ) ;
column = 0 ;
for ( ; n > 0 ; n - - , p + = size ) {
uintptr off , * bitp , shift , bits ;
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
allocated = ( ( bits & bitAllocated ) ! = 0 ) ;
for ( i = 0 ; ( uint32 ) i < size ; i + = sizeof ( void * ) ) {
if ( column = = 0 ) {
runtime_printf ( " \t " ) ;
}
if ( i = = 0 ) {
runtime_printf ( allocated ? " ( " : " [ " ) ;
runtime_printf ( " %p: " , p + i ) ;
} else {
runtime_printf ( " " ) ;
}
runtime_printf ( " %p " , * ( void * * ) ( p + i ) ) ;
if ( i + sizeof ( void * ) > = size ) {
runtime_printf ( allocated ? " ) " : " ] " ) ;
}
column + + ;
if ( column = = 8 ) {
runtime_printf ( " \n " ) ;
column = 0 ;
}
}
}
runtime_printf ( " \n " ) ;
}
// A debugging function to dump the contents of memory
void
runtime_memorydump ( void )
{
uint32 spanidx ;
2013-11-06 20:49:01 +01:00
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
2012-11-21 08:03:38 +01:00
dumpspan ( spanidx ) ;
}
}
2012-12-13 00:13:29 +01:00
2011-10-27 01:57:58 +02:00
void
runtime_gchelper ( void )
{
2014-03-03 21:14:52 +01:00
uint32 nproc ;
2014-07-19 10:53:52 +02:00
runtime_m ( ) - > traceback = 2 ;
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
// parallel mark for over gc roots
runtime_parfordo ( work . markfor ) ;
2012-12-22 02:15:33 +01:00
2012-10-23 06:31:11 +02:00
// help other threads scan secondary blocks
2014-06-07 00:37:27 +02:00
scanblock ( nil , true ) ;
2011-10-27 01:57:58 +02:00
2013-07-16 08:54:42 +02:00
bufferList [ runtime_m ( ) - > helpgc ] . busy = 0 ;
2014-03-03 21:14:52 +01:00
nproc = work . nproc ; // work.nproc can change right after we increment work.ndone
if ( runtime_xadd ( & work . ndone , + 1 ) = = nproc - 1 )
2011-10-27 01:57:58 +02:00
runtime_notewakeup ( & work . alldone ) ;
2014-07-19 10:53:52 +02:00
runtime_m ( ) - > traceback = 0 ;
2011-10-27 01:57:58 +02:00
}
2011-11-28 06:45:49 +01:00
static void
2013-11-06 20:49:01 +01:00
cachestats ( void )
{
MCache * c ;
P * p , * * pp ;
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
runtime_purgecachedstats ( c ) ;
}
}
2014-06-07 00:37:27 +02:00
static void
flushallmcaches ( void )
{
P * p , * * pp ;
MCache * c ;
// Flush MCache's to MCentral.
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
runtime_MCache_ReleaseAll ( c ) ;
}
}
2014-07-19 10:53:52 +02:00
void
runtime_updatememstats ( GCStats * stats )
2011-11-28 06:45:49 +01:00
{
2012-12-22 02:15:33 +01:00
M * mp ;
2013-11-06 20:49:01 +01:00
MSpan * s ;
2011-11-28 06:45:49 +01:00
uint32 i ;
2013-11-06 20:49:01 +01:00
uint64 stacks_inuse , smallfree ;
2012-10-23 06:31:11 +02:00
uint64 * src , * dst ;
2011-11-28 06:45:49 +01:00
2012-10-23 06:31:11 +02:00
if ( stats )
runtime_memclr ( ( byte * ) stats , sizeof ( * stats ) ) ;
2011-11-28 06:45:49 +01:00
stacks_inuse = 0 ;
2012-12-22 02:15:33 +01:00
for ( mp = runtime_allm ; mp ; mp = mp - > alllink ) {
2013-07-16 08:54:42 +02:00
//stacks_inuse += mp->stackinuse*FixedStack;
2012-10-23 06:31:11 +02:00
if ( stats ) {
2012-12-22 02:15:33 +01:00
src = ( uint64 * ) & mp - > gcstats ;
2012-10-23 06:31:11 +02:00
dst = ( uint64 * ) stats ;
for ( i = 0 ; i < sizeof ( * stats ) / sizeof ( uint64 ) ; i + + )
dst [ i ] + = src [ i ] ;
2012-12-22 02:15:33 +01:00
runtime_memclr ( ( byte * ) & mp - > gcstats , sizeof ( mp - > gcstats ) ) ;
2012-10-23 06:31:11 +02:00
}
2013-07-16 08:54:42 +02:00
}
2013-11-06 20:49:01 +01:00
mstats . stacks_inuse = stacks_inuse ;
mstats . mcache_inuse = runtime_mheap . cachealloc . inuse ;
mstats . mspan_inuse = runtime_mheap . spanalloc . inuse ;
mstats . sys = mstats . heap_sys + mstats . stacks_sys + mstats . mspan_sys +
mstats . mcache_sys + mstats . buckhash_sys + mstats . gc_sys + mstats . other_sys ;
// Calculate memory allocator stats.
// During program execution we only count number of frees and amount of freed memory.
// Current number of alive object in the heap and amount of alive heap memory
// are calculated by scanning all spans.
// Total number of mallocs is calculated as number of frees plus number of alive objects.
// Similarly, total amount of allocated memory is calculated as amount of freed memory
// plus amount of alive heap memory.
mstats . alloc = 0 ;
mstats . total_alloc = 0 ;
mstats . nmalloc = 0 ;
mstats . nfree = 0 ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . by_size [ i ] . nmalloc = 0 ;
mstats . by_size [ i ] . nfree = 0 ;
}
// Flush MCache's to MCentral.
2014-06-07 00:37:27 +02:00
flushallmcaches ( ) ;
2013-11-06 20:49:01 +01:00
// Aggregate local stats.
cachestats ( ) ;
// Scan all spans and count number of alive objects.
for ( i = 0 ; i < runtime_mheap . nspan ; i + + ) {
s = runtime_mheap . allspans [ i ] ;
if ( s - > state ! = MSpanInUse )
continue ;
if ( s - > sizeclass = = 0 ) {
mstats . nmalloc + + ;
mstats . alloc + = s - > elemsize ;
} else {
mstats . nmalloc + = s - > ref ;
mstats . by_size [ s - > sizeclass ] . nmalloc + = s - > ref ;
mstats . alloc + = s - > ref * s - > elemsize ;
2011-11-28 06:45:49 +01:00
}
}
2013-11-06 20:49:01 +01:00
// Aggregate by size class.
smallfree = 0 ;
mstats . nfree = runtime_mheap . nlargefree ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . nfree + = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nfree = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nmalloc + = runtime_mheap . nsmallfree [ i ] ;
smallfree + = runtime_mheap . nsmallfree [ i ] * runtime_class_to_size [ i ] ;
}
mstats . nmalloc + = mstats . nfree ;
// Calculate derived stats.
mstats . total_alloc = mstats . alloc + runtime_mheap . largefree + smallfree ;
mstats . heap_alloc = mstats . alloc ;
mstats . heap_objects = mstats . nmalloc - mstats . nfree ;
2011-11-28 06:45:49 +01:00
}
2012-12-13 00:13:29 +01:00
// Structure of arguments passed to function gc().
2013-11-06 20:49:01 +01:00
// This allows the arguments to be passed via runtime_mcall.
2012-12-13 00:13:29 +01:00
struct gc_args
{
2013-11-06 20:49:01 +01:00
int64 start_time ; // start time of GC in ns (just before stoptheworld)
2014-07-19 10:53:52 +02:00
bool eagersweep ;
2012-12-13 00:13:29 +01:00
} ;
static void gc ( struct gc_args * args ) ;
2013-11-06 20:49:01 +01:00
static void mgc ( G * gp ) ;
2012-12-13 00:13:29 +01:00
2013-07-16 08:54:42 +02:00
static int32
readgogc ( void )
{
2015-10-31 01:59:47 +01:00
String s ;
2013-07-16 08:54:42 +02:00
const byte * p ;
2015-10-31 01:59:47 +01:00
s = runtime_getenv ( " GOGC " ) ;
if ( s . len = = 0 )
2013-07-16 08:54:42 +02:00
return 100 ;
2015-10-31 01:59:47 +01:00
p = s . str ;
if ( s . len = = 3 & & runtime_strcmp ( ( const char * ) p , " off " ) = = 0 )
2013-07-16 08:54:42 +02:00
return - 1 ;
2015-10-31 01:59:47 +01:00
return runtime_atoi ( p , s . len ) ;
2013-07-16 08:54:42 +02:00
}
2014-07-19 10:53:52 +02:00
// force = 1 - do GC regardless of current heap usage
// force = 2 - go GC and eager sweep
2010-12-03 05:34:57 +01:00
void
2011-11-28 06:45:49 +01:00
runtime_gc ( int32 force )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2013-11-06 20:49:01 +01:00
G * g ;
struct gc_args a ;
int32 i ;
2012-10-23 06:31:11 +02:00
// The atomic operations are not atomic if the uint64s
// are not aligned on uint64 boundaries. This has been
// a problem in the past.
if ( ( ( ( uintptr ) & work . empty ) & 7 ) ! = 0 )
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2013-07-16 08:54:42 +02:00
if ( ( ( ( uintptr ) & work . full ) & 7 ) ! = 0 )
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2010-12-03 05:34:57 +01:00
2012-02-10 16:52:37 +01:00
// Make sure all registers are saved on stack so that
// scanstack sees them.
__builtin_unwind_init ( ) ;
2010-12-03 05:34:57 +01:00
// The gc is turned off (via enablegc) until
// the bootstrap has completed.
// Also, malloc gets called in the guts
// of a number of libraries that might be
// holding locks. To avoid priority inversion
// problems, don't bother trying to run gc
// while holding a lock. The next mallocgc
// without a lock will do the gc instead.
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2013-11-06 20:49:01 +01:00
if ( ! mstats . enablegc | | runtime_g ( ) = = m - > g0 | | m - > locks > 0 | | runtime_panicking )
2010-12-03 05:34:57 +01:00
return ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown ) { // first time through
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
runtime_unlock ( & runtime_mheap ) ;
2010-12-03 05:34:57 +01:00
}
if ( gcpercent < 0 )
return ;
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
2014-07-19 10:53:52 +02:00
if ( force = = 0 & & mstats . heap_alloc < mstats . next_gc ) {
2013-11-06 20:49:01 +01:00
// typically threads which lost the race to grab
// worldsema exit here when gc is done.
runtime_semrelease ( & runtime_worldsema ) ;
return ;
}
2012-12-13 00:13:29 +01:00
2013-11-06 20:49:01 +01:00
// Ok, we're doing it! Stop everybody else
a . start_time = runtime_nanotime ( ) ;
2014-07-19 10:53:52 +02:00
a . eagersweep = force > = 2 ;
2013-11-06 20:49:01 +01:00
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2014-06-07 00:37:27 +02:00
2014-06-05 01:15:33 +02:00
clearpools ( ) ;
2013-11-06 20:49:01 +01:00
// Run gc on the g0 stack. We do this so that the g stack
// we're currently running on will no longer change. Cuts
// the root set down a bit (g0 stacks are not scanned, and
// we don't need to scan gc's internal state). Also an
// enabler for copyable stacks.
for ( i = 0 ; i < ( runtime_debug . gctrace > 1 ? 2 : 1 ) ; i + + ) {
2014-07-19 10:53:52 +02:00
if ( i > 0 )
a . start_time = runtime_nanotime ( ) ;
2013-11-06 20:49:01 +01:00
// switch to g0, call gc(&a), then switch back
g = runtime_g ( ) ;
g - > param = & a ;
g - > status = Gwaiting ;
g - > waitreason = " garbage collection " ;
runtime_mcall ( mgc ) ;
2014-08-16 00:16:55 +02:00
m = runtime_m ( ) ;
2012-12-13 00:13:29 +01:00
}
2013-11-06 20:49:01 +01:00
// all done
m - > gcing = 0 ;
m - > locks + + ;
runtime_semrelease ( & runtime_worldsema ) ;
runtime_starttheworld ( ) ;
m - > locks - - ;
// now that gc is done, kick off finalizer thread if needed
2014-06-07 00:37:27 +02:00
if ( ! ConcurrentSweep ) {
// give the queued finalizers, if any, a chance to run
runtime_gosched ( ) ;
} else {
// For gccgo, let other goroutines run.
runtime_gosched ( ) ;
2013-11-06 20:49:01 +01:00
}
}
static void
mgc ( G * gp )
{
gc ( gp - > param ) ;
gp - > param = nil ;
gp - > status = Grunning ;
runtime_gogo ( gp ) ;
2012-12-13 00:13:29 +01:00
}
static void
gc ( struct gc_args * args )
{
M * m ;
2013-01-29 21:52:43 +01:00
int64 t0 , t1 , t2 , t3 , t4 ;
2014-06-07 00:37:27 +02:00
uint64 heap0 , heap1 , obj , ninstr ;
2012-12-13 00:13:29 +01:00
GCStats stats ;
uint32 i ;
2013-01-29 21:52:43 +01:00
// Eface eface;
2012-12-13 00:13:29 +01:00
m = runtime_m ( ) ;
2014-07-19 10:53:52 +02:00
if ( runtime_debug . allocfreetrace )
runtime_tracegc ( ) ;
m - > traceback = 2 ;
2013-11-06 20:49:01 +01:00
t0 = args - > start_time ;
2014-06-07 00:37:27 +02:00
work . tstart = args - > start_time ;
2010-12-03 05:34:57 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_memclr ( ( byte * ) & gcstats , sizeof ( gcstats ) ) ;
2012-12-22 02:15:33 +01:00
m - > locks + + ; // disable gc during mallocs in parforalloc
if ( work . markfor = = nil )
work . markfor = runtime_parforalloc ( MaxGcproc ) ;
m - > locks - - ;
2014-07-19 10:53:52 +02:00
t1 = 0 ;
if ( runtime_debug . gctrace )
t1 = runtime_nanotime ( ) ;
2014-06-07 00:37:27 +02:00
// Sweep what is not sweeped by bgsweep.
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 )
gcstats . npausesweep + + ;
2011-10-27 01:57:58 +02:00
work . nwait = 0 ;
work . ndone = 0 ;
2012-10-23 06:31:11 +02:00
work . nproc = runtime_gcprocs ( ) ;
2015-10-31 01:59:47 +01:00
runtime_parforsetup ( work . markfor , work . nproc , RootCount + runtime_allglen , false , & markroot_funcval ) ;
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 ) {
runtime_noteclear ( & work . alldone ) ;
runtime_helpgc ( work . nproc ) ;
}
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
t2 = 0 ;
if ( runtime_debug . gctrace )
t2 = runtime_nanotime ( ) ;
2013-01-29 21:52:43 +01:00
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
runtime_parfordo ( work . markfor ) ;
2014-06-07 00:37:27 +02:00
scanblock ( nil , true ) ;
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
t3 = 0 ;
if ( runtime_debug . gctrace )
t3 = runtime_nanotime ( ) ;
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
bufferList [ m - > helpgc ] . busy = 0 ;
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 )
runtime_notesleep ( & work . alldone ) ;
2013-11-06 20:49:01 +01:00
cachestats ( ) ;
2014-06-07 00:37:27 +02:00
// next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
// estimate what was live heap size after previous GC (for tracing only)
heap0 = mstats . next_gc * 100 / ( gcpercent + 100 ) ;
// conservatively set next_gc to high value assuming that everything is live
// concurrent/lazy sweep will reduce this number while discovering new garbage
2015-09-15 14:32:31 +02:00
mstats . next_gc = mstats . heap_alloc + ( mstats . heap_alloc - runtime_stacks_sys ) * gcpercent / 100 ;
2011-03-17 00:05:44 +01:00
2013-01-29 21:52:43 +01:00
t4 = runtime_nanotime ( ) ;
2014-07-19 10:53:52 +02:00
mstats . last_gc = runtime_unixnanotime ( ) ; // must be Unix time to make sense to user
2013-01-29 21:52:43 +01:00
mstats . pause_ns [ mstats . numgc % nelem ( mstats . pause_ns ) ] = t4 - t0 ;
2015-10-31 01:59:47 +01:00
mstats . pause_end [ mstats . numgc % nelem ( mstats . pause_end ) ] = mstats . last_gc ;
2013-01-29 21:52:43 +01:00
mstats . pause_total_ns + = t4 - t0 ;
2010-12-03 05:34:57 +01:00
mstats . numgc + + ;
if ( mstats . debuggc )
2013-01-29 21:52:43 +01:00
runtime_printf ( " pause %D \n " , t4 - t0 ) ;
2011-10-27 01:57:58 +02:00
2013-11-06 20:49:01 +01:00
if ( runtime_debug . gctrace ) {
heap1 = mstats . heap_alloc ;
2014-07-19 10:53:52 +02:00
runtime_updatememstats ( & stats ) ;
if ( heap1 ! = mstats . heap_alloc ) {
runtime_printf ( " runtime: mstats skew: heap=%D/%D \n " , heap1 , mstats . heap_alloc ) ;
runtime_throw ( " mstats skew " ) ;
}
2014-06-07 00:37:27 +02:00
obj = mstats . nmalloc - mstats . nfree ;
2013-11-06 20:49:01 +01:00
2014-06-07 00:37:27 +02:00
stats . nprocyield + = work . markfor - > nprocyield ;
stats . nosyield + = work . markfor - > nosyield ;
stats . nsleep + = work . markfor - > nsleep ;
2013-11-06 20:49:01 +01:00
2014-07-19 10:53:52 +02:00
runtime_printf ( " gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects, "
2014-06-07 00:37:27 +02:00
" %d/%d/%d sweeps, "
2012-10-23 06:31:11 +02:00
" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields \n " ,
2014-07-19 10:53:52 +02:00
mstats . numgc , work . nproc , ( t1 - t0 ) / 1000 , ( t2 - t1 ) / 1000 , ( t3 - t2 ) / 1000 , ( t4 - t3 ) / 1000 ,
2014-06-07 00:37:27 +02:00
heap0 > > 20 , heap1 > > 20 , obj ,
2012-10-23 06:31:11 +02:00
mstats . nmalloc , mstats . nfree ,
2014-06-07 00:37:27 +02:00
sweep . nspan , gcstats . nbgsweep , gcstats . npausesweep ,
2012-10-23 06:31:11 +02:00
stats . nhandoff , stats . nhandoffcnt ,
2014-06-07 00:37:27 +02:00
work . markfor - > nsteal , work . markfor - > nstealcnt ,
2012-10-23 06:31:11 +02:00
stats . nprocyield , stats . nosyield , stats . nsleep ) ;
2014-06-07 00:37:27 +02:00
gcstats . nbgsweep = gcstats . npausesweep = 0 ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_printf ( " scan: %D bytes, %D objects, %D untyped, %D types from MSpan \n " ,
gcstats . nbytes , gcstats . obj . cnt , gcstats . obj . notype , gcstats . obj . typelookup ) ;
if ( gcstats . ptr . cnt ! = 0 )
runtime_printf ( " avg ptrbufsize: %D (%D/%D) \n " ,
gcstats . ptr . sum / gcstats . ptr . cnt , gcstats . ptr . sum , gcstats . ptr . cnt ) ;
if ( gcstats . obj . cnt ! = 0 )
runtime_printf ( " avg nobj: %D (%D/%D) \n " ,
gcstats . obj . sum / gcstats . obj . cnt , gcstats . obj . sum , gcstats . obj . cnt ) ;
runtime_printf ( " rescans: %D, %D bytes \n " , gcstats . rescan , gcstats . rescanbytes ) ;
runtime_printf ( " instruction counts: \n " ) ;
ninstr = 0 ;
for ( i = 0 ; i < nelem ( gcstats . instr ) ; i + + ) {
runtime_printf ( " \t %d: \t %D \n " , i , gcstats . instr [ i ] ) ;
ninstr + = gcstats . instr [ i ] ;
}
runtime_printf ( " \t total: \t %D \n " , ninstr ) ;
runtime_printf ( " putempty: %D, getfull: %D \n " , gcstats . putempty , gcstats . getfull ) ;
2013-11-06 20:49:01 +01:00
runtime_printf ( " markonly base lookup: bit %D word %D span %D \n " , gcstats . markonly . foundbit , gcstats . markonly . foundword , gcstats . markonly . foundspan ) ;
runtime_printf ( " flushptrbuf base lookup: bit %D word %D span %D \n " , gcstats . flushptrbuf . foundbit , gcstats . flushptrbuf . foundword , gcstats . flushptrbuf . foundspan ) ;
2013-07-16 08:54:42 +02:00
}
2011-03-17 00:05:44 +01:00
}
2012-10-23 06:31:11 +02:00
2014-06-07 00:37:27 +02:00
// We cache current runtime_mheap.allspans array in sweep.spans,
// because the former can be resized and freed.
// Otherwise we would need to take heap lock every time
// we want to convert span index to span pointer.
// Free the old cached array if necessary.
if ( sweep . spans & & sweep . spans ! = runtime_mheap . allspans )
runtime_SysFree ( sweep . spans , sweep . nspan * sizeof ( sweep . spans [ 0 ] ) , & mstats . other_sys ) ;
// Cache the current array.
runtime_mheap . sweepspans = runtime_mheap . allspans ;
runtime_mheap . sweepgen + = 2 ;
runtime_mheap . sweepdone = false ;
sweep . spans = runtime_mheap . allspans ;
sweep . nspan = runtime_mheap . nspan ;
sweep . spanidx = 0 ;
// Temporary disable concurrent sweep, because we see failures on builders.
2014-07-19 10:53:52 +02:00
if ( ConcurrentSweep & & ! args - > eagersweep ) {
2014-06-07 00:37:27 +02:00
runtime_lock ( & gclock ) ;
if ( sweep . g = = nil )
sweep . g = __go_go ( bgsweep , nil ) ;
else if ( sweep . parked ) {
sweep . parked = false ;
runtime_ready ( sweep . g ) ;
}
runtime_unlock ( & gclock ) ;
} else {
// Sweep all spans eagerly.
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 )
gcstats . npausesweep + + ;
2015-01-15 01:27:56 +01:00
// Do an additional mProf_GC, because all 'free' events are now real as well.
runtime_MProf_GC ( ) ;
2014-06-07 00:37:27 +02:00
}
2012-03-02 21:01:37 +01:00
runtime_MProf_GC ( ) ;
2014-07-19 10:53:52 +02:00
m - > traceback = 0 ;
2010-12-03 05:34:57 +01:00
}
2014-06-07 00:37:27 +02:00
extern uintptr runtime_sizeof_C_MStats
__asm__ ( GOSYM_PREFIX " runtime.Sizeof_C_MStats " ) ;
2012-02-09 09:19:58 +01:00
void runtime_ReadMemStats ( MStats * )
2013-01-24 20:44:23 +01:00
__asm__ ( GOSYM_PREFIX " runtime.ReadMemStats " ) ;
2011-09-16 17:47:21 +02:00
void
2012-02-09 09:19:58 +01:00
runtime_ReadMemStats ( MStats * stats )
2011-09-16 17:47:21 +02:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2012-03-02 21:01:37 +01:00
// Have to acquire worldsema to stop the world,
2011-09-16 17:47:21 +02:00
// because stoptheworld can only be used by
// one goroutine at a time, and there might be
// a pending garbage collection already calling it.
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2011-09-16 17:47:21 +02:00
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2014-07-19 10:53:52 +02:00
runtime_updatememstats ( nil ) ;
2014-06-07 00:37:27 +02:00
// Size of the trailing by_size array differs between Go and C,
// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
runtime_memmove ( stats , & mstats , runtime_sizeof_C_MStats ) ;
2011-09-16 17:47:21 +02:00
m - > gcing = 0 ;
2013-11-06 20:49:01 +01:00
m - > locks + + ;
2012-03-02 21:01:37 +01:00
runtime_semrelease ( & runtime_worldsema ) ;
2012-10-23 06:31:11 +02:00
runtime_starttheworld ( ) ;
2013-11-06 20:49:01 +01:00
m - > locks - - ;
2011-09-16 17:47:21 +02:00
}
2013-07-16 08:54:42 +02:00
void runtime_debug_readGCStats ( Slice * )
__asm__ ( " runtime_debug.readGCStats " ) ;
void
runtime_debug_readGCStats ( Slice * pauses )
{
uint64 * p ;
uint32 i , n ;
// Calling code in runtime/debug should make the slice large enough.
if ( ( size_t ) pauses - > cap < nelem ( mstats . pause_ns ) + 3 )
runtime_throw ( " runtime: short slice passed to readGCStats " ) ;
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = ( uint64 * ) pauses - > array ;
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
n = mstats . numgc ;
if ( n > nelem ( mstats . pause_ns ) )
n = nelem ( mstats . pause_ns ) ;
// The pause buffer is circular. The most recent pause is at
// pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
// from there to go back farther in time. We deliver the times
// most recent first (in p[0]).
for ( i = 0 ; i < n ; i + + )
p [ i ] = mstats . pause_ns [ ( mstats . numgc - 1 - i ) % nelem ( mstats . pause_ns ) ] ;
p [ n ] = mstats . last_gc ;
p [ n + 1 ] = mstats . numgc ;
p [ n + 2 ] = mstats . pause_total_ns ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
pauses - > __count = n + 3 ;
}
2014-07-12 02:01:09 +02:00
int32
runtime_setgcpercent ( int32 in ) {
int32 out ;
2013-07-16 08:54:42 +02:00
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
out = gcpercent ;
if ( in < 0 )
in = - 1 ;
gcpercent = in ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
return out ;
}
static void
gchelperstart ( void )
{
M * m ;
m = runtime_m ( ) ;
if ( m - > helpgc < 0 | | m - > helpgc > = MaxGcproc )
runtime_throw ( " gchelperstart: bad m->helpgc " ) ;
if ( runtime_xchg ( & bufferList [ m - > helpgc ] . busy , 1 ) )
runtime_throw ( " gchelperstart: already busy " ) ;
2013-11-06 20:49:01 +01:00
if ( runtime_g ( ) ! = m - > g0 )
runtime_throw ( " gchelper not running on g0 stack " ) ;
2013-07-16 08:54:42 +02:00
}
2010-12-03 05:34:57 +01:00
static void
2011-11-28 06:45:49 +01:00
runfinq ( void * dummy __attribute__ ( ( unused ) ) )
2010-12-03 05:34:57 +01:00
{
2011-10-27 01:57:58 +02:00
Finalizer * f ;
FinBlock * fb , * next ;
uint32 i ;
2013-11-06 20:49:01 +01:00
Eface ef ;
Iface iface ;
2010-12-03 05:34:57 +01:00
2014-07-19 10:53:52 +02:00
// This function blocks for long periods of time, and because it is written in C
// we have no liveness information. Zero everything so that uninitialized pointers
// do not cause memory leaks.
f = nil ;
fb = nil ;
next = nil ;
i = 0 ;
ef . __type_descriptor = nil ;
ef . __object = nil ;
// force flush to memory
USED ( & f ) ;
USED ( & fb ) ;
USED ( & next ) ;
USED ( & i ) ;
USED ( & ef ) ;
2010-12-03 05:34:57 +01:00
for ( ; ; ) {
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
fb = finq ;
2010-12-03 05:34:57 +01:00
finq = nil ;
2011-10-27 01:57:58 +02:00
if ( fb = = nil ) {
2014-07-19 10:53:52 +02:00
runtime_fingwait = true ;
runtime_g ( ) - > isbackground = true ;
runtime_parkunlock ( & finlock , " finalizer wait " ) ;
runtime_g ( ) - > isbackground = false ;
2010-12-03 05:34:57 +01:00
continue ;
}
2014-07-19 10:53:52 +02:00
runtime_unlock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
for ( ; fb ; fb = next ) {
next = fb - > next ;
for ( i = 0 ; i < ( uint32 ) fb - > cnt ; i + + ) {
2013-11-06 20:49:01 +01:00
const Type * fint ;
2013-09-03 23:52:37 +02:00
void * param ;
2011-10-27 01:57:58 +02:00
f = & fb - > fin [ i ] ;
2013-11-06 20:49:01 +01:00
fint = ( ( const Type * * ) f - > ft - > __in . array ) [ 0 ] ;
2015-01-15 01:27:56 +01:00
if ( ( fint - > __code & kindMask ) = = KindPtr ) {
2013-11-06 20:49:01 +01:00
// direct use of pointer
param = & f - > arg ;
} else if ( ( ( const InterfaceType * ) fint ) - > __methods . __count = = 0 ) {
// convert to empty interface
2014-06-07 00:37:27 +02:00
ef . __type_descriptor = ( const Type * ) f - > ot ;
2013-11-06 20:49:01 +01:00
ef . __object = f - > arg ;
param = & ef ;
} else {
// convert to interface with methods
iface . __methods = __go_convert_interface_2 ( ( const Type * ) fint ,
( const Type * ) f - > ot ,
1 ) ;
iface . __object = f - > arg ;
if ( iface . __methods = = nil )
runtime_throw ( " invalid type conversion in runfinq " ) ;
param = & iface ;
}
2013-09-03 23:52:37 +02:00
reflect_call ( f - > ft , f - > fn , 0 , 0 , & param , nil ) ;
2011-10-27 01:57:58 +02:00
f - > fn = nil ;
f - > arg = nil ;
2013-11-06 20:49:01 +01:00
f - > ot = nil ;
2011-10-27 01:57:58 +02:00
}
fb - > cnt = 0 ;
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
fb - > next = finc ;
finc = fb ;
2014-07-19 10:53:52 +02:00
runtime_unlock ( & finlock ) ;
2010-12-03 05:34:57 +01:00
}
2014-07-19 10:53:52 +02:00
// Zero everything that's dead, to avoid memory leaks.
// See comment at top of function.
f = nil ;
fb = nil ;
next = nil ;
i = 0 ;
ef . __type_descriptor = nil ;
ef . __object = nil ;
2010-12-03 05:34:57 +01:00
runtime_gc ( 1 ) ; // trigger another gc to clean up the finalized objects, if possible
}
}
2014-07-19 10:53:52 +02:00
void
runtime_createfing ( void )
{
if ( fing ! = nil )
return ;
// Here we use gclock instead of finlock,
// because newproc1 can allocate, which can cause on-demand span sweep,
// which can queue finalizers, which would deadlock.
runtime_lock ( & gclock ) ;
if ( fing = = nil )
fing = __go_go ( runfinq , nil ) ;
runtime_unlock ( & gclock ) ;
}
G *
runtime_wakefing ( void )
{
G * res ;
res = nil ;
runtime_lock ( & finlock ) ;
if ( runtime_fingwait & & runtime_fingwake ) {
runtime_fingwait = false ;
runtime_fingwake = false ;
res = fing ;
}
runtime_unlock ( & finlock ) ;
return res ;
}
2011-03-17 00:05:44 +01:00
void
2014-06-05 01:15:33 +02:00
runtime_marknogc ( void * v )
2011-03-17 00:05:44 +01:00
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b = ( * b & ~ ( bitAllocated < < shift ) ) | bitBlockBoundary < < shift ;
2014-06-05 01:15:33 +02:00
}
void
runtime_markscan ( void * v )
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b | = bitScan < < shift ;
2011-03-17 00:05:44 +01:00
}
2014-07-19 10:53:52 +02:00
// mark the block at v as freed.
2011-03-17 00:05:44 +01:00
void
2014-07-19 10:53:52 +02:00
runtime_markfreed ( void * v )
2011-03-17 00:05:44 +01:00
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2012-05-24 22:44:34 +02:00
if ( 0 )
2014-07-19 10:53:52 +02:00
runtime_printf ( " markfreed %p \n " , v ) ;
2011-03-17 00:05:44 +01:00
2014-07-19 10:53:52 +02:00
if ( ( byte * ) v > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2013-11-06 20:49:01 +01:00
runtime_throw ( " markfreed: bad pointer " ) ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b = ( * b & ~ ( bitMask < < shift ) ) | ( bitAllocated < < shift ) ;
2011-03-17 00:05:44 +01:00
}
// check that the block at v of size n is marked freed.
void
runtime_checkfreed ( void * v , uintptr n )
{
uintptr * b , bits , off , shift ;
if ( ! runtime_checking )
return ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
return ; // not allocated, so okay
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
bits = * b > > shift ;
if ( ( bits & bitAllocated ) ! = 0 ) {
runtime_printf ( " checkfreed %p+%p: off=%p have=%p \n " ,
2012-05-24 22:44:34 +02:00
v , n , off , bits & bitMask ) ;
2011-03-17 00:05:44 +01:00
runtime_throw ( " checkfreed: not freed " ) ;
}
}
// mark the span of memory at v as having n blocks of the given size.
// if leftover is true, there is left over space at the end of the span.
void
runtime_markspan ( void * v , uintptr size , uintptr n , bool leftover )
{
2014-07-19 10:53:52 +02:00
uintptr * b , * b0 , off , shift , i , x ;
2011-03-17 00:05:44 +01:00
byte * p ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + size * n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
2014-06-05 01:15:33 +02:00
if ( runtime_checking ) {
// bits should be all zero at the start
off = ( byte * ) v + size - runtime_mheap . arena_start ;
b = ( uintptr * ) ( runtime_mheap . arena_start - off / wordsPerBitmapWord ) ;
for ( i = 0 ; i < size / PtrSize / wordsPerBitmapWord ; i + + ) {
if ( b [ i ] ! = 0 )
runtime_throw ( " markspan: span bits not zero " ) ;
}
}
2011-03-17 00:05:44 +01:00
p = v ;
if ( leftover ) // mark a boundary just past end of last block too
n + + ;
2014-07-19 10:53:52 +02:00
b0 = nil ;
x = 0 ;
2011-03-17 00:05:44 +01:00
for ( ; n - - > 0 ; p + = size ) {
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
if ( b0 ! = b ) {
if ( b0 ! = nil )
* b0 = x ;
b0 = b ;
x = 0 ;
}
x | = bitAllocated < < shift ;
2011-03-17 00:05:44 +01:00
}
2014-07-19 10:53:52 +02:00
* b0 = x ;
2011-03-17 00:05:44 +01:00
}
// unmark the span of memory at v of length n bytes.
void
runtime_unmarkspan ( void * v , uintptr n )
{
uintptr * p , * b , off ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
p = v ;
2013-11-06 20:49:01 +01:00
off = p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
2011-03-17 00:05:44 +01:00
if ( off % wordsPerBitmapWord ! = 0 )
runtime_throw ( " markspan: unaligned pointer " ) ;
2013-11-06 20:49:01 +01:00
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
n / = PtrSize ;
if ( n % wordsPerBitmapWord ! = 0 )
runtime_throw ( " unmarkspan: unaligned length " ) ;
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
n / = wordsPerBitmapWord ;
while ( n - - > 0 )
* b - - = 0 ;
}
void
runtime_MHeap_MapBits ( MHeap * h )
{
2012-04-20 06:58:26 +02:00
size_t page_size ;
2011-03-17 00:05:44 +01:00
// Caller has added extra mappings to the arena.
// Add extra mappings of bitmap words as needed.
// We allocate extra bitmap pieces in chunks of bitmapChunk.
enum {
bitmapChunk = 8192
} ;
uintptr n ;
2011-10-27 01:57:58 +02:00
2011-03-17 00:05:44 +01:00
n = ( h - > arena_used - h - > arena_start ) / wordsPerBitmapWord ;
2013-11-06 20:49:01 +01:00
n = ROUND ( n , bitmapChunk ) ;
2014-07-19 10:53:52 +02:00
n = ROUND ( n , PageSize ) ;
page_size = getpagesize ( ) ;
n = ROUND ( n , page_size ) ;
2011-03-17 00:05:44 +01:00
if ( h - > bitmap_mapped > = n )
return ;
2014-07-19 10:53:52 +02:00
runtime_SysMap ( h - > arena_start - n , n - h - > bitmap_mapped , h - > arena_reserved , & mstats . gc_sys ) ;
2011-03-17 00:05:44 +01:00
h - > bitmap_mapped = n ;
}
2015-10-31 01:59:47 +01:00
// typedmemmove copies a value of type t to dst from src.
extern void typedmemmove ( const Type * td , void * dst , const void * src )
__asm__ ( GOSYM_PREFIX " reflect.typedmemmove " ) ;
void
typedmemmove ( const Type * td , void * dst , const void * src )
{
runtime_memmove ( dst , src , td - > __size ) ;
}
// typedslicecopy copies a slice of elemType values from src to dst,
// returning the number of elements copied.
extern intgo typedslicecopy ( const Type * elem , Slice dst , Slice src )
__asm__ ( GOSYM_PREFIX " reflect.typedslicecopy " ) ;
intgo
typedslicecopy ( const Type * elem , Slice dst , Slice src )
{
intgo n ;
void * dstp ;
void * srcp ;
n = dst . __count ;
if ( n > src . __count )
n = src . __count ;
if ( n = = 0 )
return 0 ;
dstp = dst . __values ;
srcp = src . __values ;
memmove ( dstp , srcp , ( uintptr_t ) n * elem - > __size ) ;
return n ;
}