2014-09-20 22:42:59 +02:00
// Copyright 2009 The Go Authors. All rights reserved.
2010-12-03 05:34:57 +01:00
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
2014-06-07 00:37:27 +02:00
// Garbage collector (GC).
//
// GC is:
// - mark&sweep
// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
// - parallel (up to MaxGcproc threads)
// - partially concurrent (mark is stop-the-world, while sweep is concurrent)
// - non-moving/non-compacting
// - full (non-partial)
//
// GC rate.
// Next GC is after we've allocated an extra amount of memory proportional to
// the amount already in use. The proportion is controlled by GOGC environment variable
// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
// (and also the amount of extra memory used).
//
// Concurrent sweep.
// The sweep phase proceeds concurrently with normal program execution.
// The heap is swept span-by-span both lazily (when a goroutine needs another span)
// and concurrently in a background goroutine (this helps programs that are not CPU bound).
// However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
// and so next_gc calculation is tricky and happens as follows.
// At the end of the stop-the-world phase next_gc is conservatively set based on total
// heap size; all spans are marked as "needs sweeping".
// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
// closer to the target value. However, this is not enough to avoid over-allocating memory.
// Consider that a goroutine wants to allocate a new span for a large object and
// there are no free swept spans, but there are small-object unswept spans.
// If the goroutine naively allocates a new span, it can surpass the yet-unknown
// target next_gc value. In order to prevent such cases (1) when a goroutine needs
// to allocate a new small-object span, it sweeps small-object spans for the same
// object size until it frees at least one object; (2) when a goroutine needs to
// allocate large-object span from heap, it sweeps spans until it frees at least
// that many pages into heap. Together these two measures ensure that we don't surpass
// target next_gc value by a large margin. There is an exception: if a goroutine sweeps
// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
// but there can still be other one-page unswept spans which could be combined into a two-page span.
// It's critical to ensure that no operations proceed on unswept spans (that would corrupt
// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
// When a goroutine explicitly frees an object or sets a finalizer, it ensures that
// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
// The finalizer goroutine is kicked off only when all spans are swept.
// When the next GC starts, it sweeps all not-yet-swept spans (if any).
2010-12-03 05:34:57 +01:00
2012-04-20 06:58:26 +02:00
# include <unistd.h>
2010-12-03 05:34:57 +01:00
# include "runtime.h"
2011-10-27 01:57:58 +02:00
# include "arch.h"
2010-12-03 05:34:57 +01:00
# include "malloc.h"
2012-12-22 02:15:33 +01:00
# include "mgc0.h"
2014-07-19 10:53:52 +02:00
# include "chan.h"
2013-01-29 21:52:43 +01:00
# include "go-type.h"
// Map gccgo field names to gc field names.
// Slice aka __go_open_array.
# define array __values
# define cap __capacity
// Iface aka __go_interface
# define tab __methods
2013-07-16 08:54:42 +02:00
// Hmap aka __go_map
typedef struct __go_map Hmap ;
2013-01-29 21:52:43 +01:00
// Type aka __go_type_descriptor
2013-07-16 08:54:42 +02:00
# define string __reflection
2013-01-29 21:52:43 +01:00
// PtrType aka __go_ptr_type
# define elem __element_type
2010-12-03 05:34:57 +01:00
2011-11-28 06:45:49 +01:00
# ifdef USING_SPLIT_STACK
extern void * __splitstack_find ( void * , void * , size_t * , void * * , void * * ,
void * * ) ;
extern void * __splitstack_find_context ( void * context [ 10 ] , size_t * , void * * ,
void * * , void * * ) ;
# endif
2010-12-03 05:34:57 +01:00
enum {
2011-03-17 00:05:44 +01:00
Debug = 0 ,
2013-07-16 08:54:42 +02:00
CollectStats = 0 ,
2014-06-07 00:37:27 +02:00
ConcurrentSweep = 1 ,
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
WorkbufSize = 16 * 1024 ,
FinBlockSize = 4 * 1024 ,
2012-12-22 02:15:33 +01:00
handoffThreshold = 4 ,
IntermediateBufferCapacity = 64 ,
2013-01-29 21:52:43 +01:00
// Bits in type information
PRECISE = 1 ,
LOOP = 2 ,
PC_BITS = PRECISE | LOOP ,
2013-11-06 20:49:01 +01:00
2014-06-07 00:37:27 +02:00
RootData = 0 ,
RootBss = 1 ,
RootFinalizers = 2 ,
RootSpanTypes = 3 ,
RootFlushCaches = 4 ,
RootCount = 5 ,
2010-12-03 05:34:57 +01:00
} ;
2014-06-07 00:37:27 +02:00
# define GcpercentUnknown (-2)
// Initialized from $GOGC. GOGC=off means no gc.
static int32 gcpercent = GcpercentUnknown ;
2014-07-19 10:53:52 +02:00
static FuncVal * poolcleanup ;
2014-06-05 01:15:33 +02:00
2014-07-19 10:53:52 +02:00
void sync_runtime_registerPoolCleanup ( FuncVal * )
__asm__ ( GOSYM_PREFIX " sync.runtime_registerPoolCleanup " ) ;
2014-06-05 01:15:33 +02:00
void
2014-07-19 10:53:52 +02:00
sync_runtime_registerPoolCleanup ( FuncVal * f )
2014-06-05 01:15:33 +02:00
{
2014-07-19 10:53:52 +02:00
poolcleanup = f ;
2014-06-05 01:15:33 +02:00
}
static void
clearpools ( void )
{
2014-06-07 00:37:27 +02:00
P * p , * * pp ;
MCache * c ;
// clear sync.Pool's
2014-07-19 10:53:52 +02:00
if ( poolcleanup ! = nil ) {
2015-01-16 23:58:53 +01:00
__builtin_call_with_static_chain ( poolcleanup - > fn ( ) ,
poolcleanup ) ;
2014-06-05 01:15:33 +02:00
}
2014-06-07 00:37:27 +02:00
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
// clear tinyalloc pool
c = p - > mcache ;
if ( c ! = nil ) {
c - > tiny = nil ;
c - > tinysize = 0 ;
}
// clear defer pools
p - > deferpool = nil ;
}
2014-06-05 01:15:33 +02:00
}
2012-03-02 21:01:37 +01:00
// Holding worldsema grants an M the right to try to stop the world.
// The procedure is:
//
// runtime_semacquire(&runtime_worldsema);
// m->gcing = 1;
// runtime_stoptheworld();
//
// ... do stuff ...
//
// m->gcing = 0;
// runtime_semrelease(&runtime_worldsema);
// runtime_starttheworld();
//
uint32 runtime_worldsema = 1 ;
2011-03-17 00:05:44 +01:00
typedef struct Workbuf Workbuf ;
struct Workbuf
2010-12-03 05:34:57 +01:00
{
2014-06-07 00:37:27 +02:00
# define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr))
2012-12-22 02:15:33 +01:00
LFNode node ; // must be first
2011-10-27 01:57:58 +02:00
uintptr nobj ;
2012-12-22 02:15:33 +01:00
Obj obj [ SIZE / sizeof ( Obj ) - 1 ] ;
uint8 _padding [ SIZE % sizeof ( Obj ) + sizeof ( Obj ) ] ;
# undef SIZE
2011-10-27 01:57:58 +02:00
} ;
typedef struct Finalizer Finalizer ;
struct Finalizer
{
2013-06-19 01:49:49 +02:00
FuncVal * fn ;
2011-10-27 01:57:58 +02:00
void * arg ;
const struct __go_func_type * ft ;
2014-09-04 00:56:09 +02:00
const PtrType * ot ;
2011-10-27 01:57:58 +02:00
} ;
typedef struct FinBlock FinBlock ;
struct FinBlock
{
FinBlock * alllink ;
FinBlock * next ;
int32 cnt ;
int32 cap ;
Finalizer fin [ 1 ] ;
2010-12-03 05:34:57 +01:00
} ;
2014-07-19 10:53:52 +02:00
static Lock finlock ; // protects the following variables
static FinBlock * finq ; // list of finalizers that are to be executed
static FinBlock * finc ; // cache of free blocks
static FinBlock * allfin ; // list of all blocks
bool runtime_fingwait ;
bool runtime_fingwake ;
2014-06-07 00:37:27 +02:00
static Lock gclock ;
2014-07-19 10:53:52 +02:00
static G * fing ;
2010-12-03 05:34:57 +01:00
2014-06-07 00:37:27 +02:00
static void runfinq ( void * ) ;
static void bgsweep ( void * ) ;
2011-03-17 00:05:44 +01:00
static Workbuf * getempty ( Workbuf * ) ;
static Workbuf * getfull ( Workbuf * ) ;
2011-10-27 01:57:58 +02:00
static void putempty ( Workbuf * ) ;
static Workbuf * handoff ( Workbuf * ) ;
2013-07-16 08:54:42 +02:00
static void gchelperstart ( void ) ;
2014-06-07 00:37:27 +02:00
static void flushallmcaches ( void ) ;
static void addstackroots ( G * gp , Workbuf * * wbufp ) ;
2011-10-27 01:57:58 +02:00
static struct {
2012-10-23 06:31:11 +02:00
uint64 full ; // lock-free list of full blocks
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
uint64 wempty ; // lock-free list of empty blocks
2012-10-23 06:31:11 +02:00
byte pad0 [ CacheLineSize ] ; // prevents false-sharing between full/empty and nproc/nwait
2011-10-27 01:57:58 +02:00
uint32 nproc ;
2014-06-07 00:37:27 +02:00
int64 tstart ;
2011-10-27 01:57:58 +02:00
volatile uint32 nwait ;
volatile uint32 ndone ;
Note alldone ;
2012-10-23 06:31:11 +02:00
ParFor * markfor ;
2011-10-27 01:57:58 +02:00
Lock ;
byte * chunk ;
uintptr nchunk ;
2014-01-17 23:43:03 +01:00
} work __attribute__ ( ( aligned ( 8 ) ) ) ;
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR ,
2013-07-16 08:54:42 +02:00
GC_CHAN ,
GC_NUM_INSTR2
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static struct {
struct {
uint64 sum ;
uint64 cnt ;
} ptr ;
uint64 nbytes ;
struct {
uint64 sum ;
uint64 cnt ;
uint64 notype ;
uint64 typelookup ;
} obj ;
uint64 rescan ;
uint64 rescanbytes ;
uint64 instr [ GC_NUM_INSTR2 ] ;
uint64 putempty ;
uint64 getfull ;
2013-11-06 20:49:01 +01:00
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} flushptrbuf ;
struct {
uint64 foundbit ;
uint64 foundword ;
uint64 foundspan ;
} markonly ;
2014-06-07 00:37:27 +02:00
uint32 nbgsweep ;
uint32 npausesweep ;
2013-07-16 08:54:42 +02:00
} gcstats ;
// markonly marks an object. It returns true if the object
// has been marked by this function, false otherwise.
// This function doesn't append the object to any buffer.
static bool
2014-07-19 10:53:52 +02:00
markonly ( const void * obj )
2013-07-16 08:54:42 +02:00
{
byte * p ;
2013-11-06 20:49:01 +01:00
uintptr * bitp , bits , shift , x , xbits , off , j ;
2013-07-16 08:54:42 +02:00
MSpan * s ;
PageID k ;
// Words outside the arena cannot be pointers.
2014-07-19 10:53:52 +02:00
if ( ( const byte * ) obj < runtime_mheap . arena_start | | ( const byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return false ;
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
2014-07-19 10:53:52 +02:00
obj = ( const void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
2013-07-16 08:54:42 +02:00
// Find bits for this word.
2014-07-19 10:53:52 +02:00
off = ( const uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
2013-11-06 20:49:01 +01:00
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
// Pointing at the beginning of a block?
2013-11-06 20:49:01 +01:00
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundbit , 1 ) ;
2013-07-16 08:54:42 +02:00
goto found ;
2013-11-06 20:49:01 +01:00
}
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
shift = j ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundword , 1 ) ;
goto found ;
}
}
2013-07-16 08:54:42 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) runtime_mheap . arena_start > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
2016-08-30 23:07:47 +02:00
if ( s = = nil | | k < s - > start | | ( uintptr ) obj > = s - > limit | | s - > state ! = MSpanInUse )
2013-07-16 08:54:42 +02:00
return false ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
uintptr size = s - > elemsize ;
2014-07-19 10:53:52 +02:00
int32 i = ( ( const byte * ) obj - p ) / size ;
2013-07-16 08:54:42 +02:00
obj = p + i * size ;
}
// Now that we know the object header, reload bits.
2014-07-19 10:53:52 +02:00
off = ( const uintptr * ) obj - ( uintptr * ) runtime_mheap . arena_start ;
2013-11-06 20:49:01 +01:00
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2013-07-16 08:54:42 +02:00
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . markonly . foundspan , 1 ) ;
2013-07-16 08:54:42 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
return false ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
return false ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
}
}
// The object is now marked
return true ;
}
// PtrTarget is a structure used by intermediate buffers.
2012-12-22 02:15:33 +01:00
// The intermediate buffers hold GC data before it
// is moved/flushed to the work buffer (Workbuf).
// The size of an intermediate buffer is very small,
// such as 32 or 64 elements.
2013-01-29 21:52:43 +01:00
typedef struct PtrTarget PtrTarget ;
2012-12-22 02:15:33 +01:00
struct PtrTarget
{
void * p ;
uintptr ti ;
} ;
2014-06-05 01:15:33 +02:00
typedef struct Scanbuf Scanbuf ;
struct Scanbuf
{
struct {
PtrTarget * begin ;
PtrTarget * end ;
PtrTarget * pos ;
} ptr ;
struct {
Obj * begin ;
Obj * end ;
Obj * pos ;
} obj ;
Workbuf * wbuf ;
Obj * wp ;
uintptr nobj ;
} ;
2013-01-29 21:52:43 +01:00
typedef struct BufferList BufferList ;
2012-12-22 02:15:33 +01:00
struct BufferList
{
2013-01-29 21:52:43 +01:00
PtrTarget ptrtarget [ IntermediateBufferCapacity ] ;
2013-07-16 08:54:42 +02:00
Obj obj [ IntermediateBufferCapacity ] ;
uint32 busy ;
byte pad [ CacheLineSize ] ;
2012-12-22 02:15:33 +01:00
} ;
2013-07-16 08:54:42 +02:00
static BufferList bufferList [ MaxGcproc ] ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
static void enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj ) ;
2012-12-22 02:15:33 +01:00
// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
// while the work buffer contains blocks which have been marked
// and are prepared to be scanned by the garbage collector.
//
// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
//
// A simplified drawing explaining how the todo-list moves from a structure to another:
//
// scanblock
// (find pointers)
// Obj ------> PtrTarget (pointer targets)
// ↑ |
2013-07-16 08:54:42 +02:00
// | |
// `----------'
// flushptrbuf
// (find block start, mark and enqueue)
2010-12-03 05:34:57 +01:00
static void
2014-06-05 01:15:33 +02:00
flushptrbuf ( Scanbuf * sbuf )
2010-12-03 05:34:57 +01:00
{
2012-12-22 02:15:33 +01:00
byte * p , * arena_start , * obj ;
2013-01-29 21:52:43 +01:00
uintptr size , * bitp , bits , shift , j , x , xbits , off , nobj , ti , n ;
2011-03-17 00:05:44 +01:00
MSpan * s ;
PageID k ;
2012-12-22 02:15:33 +01:00
Obj * wp ;
2011-03-17 00:05:44 +01:00
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
PtrTarget * ptrbuf ;
2013-01-29 21:52:43 +01:00
PtrTarget * ptrbuf_end ;
2011-05-20 02:18:15 +02:00
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
ptrbuf = sbuf - > ptr . begin ;
ptrbuf_end = sbuf - > ptr . pos ;
n = ptrbuf_end - sbuf - > ptr . begin ;
sbuf - > ptr . pos = sbuf - > ptr . begin ;
2011-03-17 00:05:44 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . ptr . sum , n ) ;
runtime_xadd64 ( & gcstats . ptr . cnt , 1 ) ;
}
2012-12-22 02:15:33 +01:00
// If buffer is nearly full, get a new one.
if ( wbuf = = nil | | nobj + n > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
if ( n > = nelem ( wbuf - > obj ) )
runtime_throw ( " ptrbuf has to be smaller than WorkBuf " ) ;
2011-03-17 00:05:44 +01:00
}
2010-12-03 05:34:57 +01:00
2014-06-05 01:15:33 +02:00
while ( ptrbuf < ptrbuf_end ) {
obj = ptrbuf - > p ;
ti = ptrbuf - > ti ;
ptrbuf + + ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if ( Debug > 1 ) {
if ( obj < runtime_mheap . arena_start | | obj > = runtime_mheap . arena_used )
runtime_throw ( " object is outside of mheap " ) ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
if ( ( ( uintptr ) obj & ( ( uintptr ) PtrSize - 1 ) ) ! = 0 ) {
obj = ( void * ) ( ( uintptr ) obj & ~ ( ( uintptr ) PtrSize - 1 ) ) ;
ti = 0 ;
}
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Find bits for this word.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
// Pointing at the beginning of a block?
if ( ( bits & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundbit , 1 ) ;
goto found ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
ti = 0 ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for ( j = shift ; j - - > 0 ; ) {
if ( ( ( xbits > > j ) & ( bitAllocated | bitBlockBoundary ) ) ! = 0 ) {
obj = ( byte * ) obj - ( shift - j ) * PtrSize ;
shift = j ;
bits = xbits > > shift ;
2013-11-06 20:49:01 +01:00
if ( CollectStats )
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . flushptrbuf . foundword , 1 ) ;
2011-03-17 00:05:44 +01:00
goto found ;
2013-11-06 20:49:01 +01:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = ( uintptr ) obj > > PageShift ;
x = k ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
2016-08-30 23:07:47 +02:00
if ( s = = nil | | k < s - > start | | ( uintptr ) obj > = s - > limit | | s - > state ! = MSpanInUse )
2014-06-05 01:15:33 +02:00
continue ;
p = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass = = 0 ) {
obj = p ;
} else {
size = s - > elemsize ;
int32 i = ( ( byte * ) obj - p ) / size ;
obj = p + i * size ;
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Now that we know the object header, reload bits.
off = ( uintptr * ) obj - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
xbits = * bitp ;
bits = xbits > > shift ;
if ( CollectStats )
runtime_xadd64 ( & gcstats . flushptrbuf . foundspan , 1 ) ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
found :
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if ( ( bits & ( bitAllocated | bitMarked ) ) ! = bitAllocated )
continue ;
if ( work . nproc = = 1 )
* bitp | = bitMarked < < shift ;
else {
for ( ; ; ) {
x = * bitp ;
if ( x & ( bitMarked < < shift ) )
goto continue_obj ;
if ( runtime_casp ( ( void * * ) bitp , ( void * ) x , ( void * ) ( x | ( bitMarked < < shift ) ) ) )
break ;
2013-07-16 08:54:42 +02:00
}
2014-06-05 01:15:33 +02:00
}
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// If object has no pointers, don't need to scan further.
if ( ( bits & bitScan ) = = 0 )
continue ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = ( uintptr ) obj > > PageShift ;
x - = ( uintptr ) arena_start > > PageShift ;
s = runtime_mheap . spans [ x ] ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
PREFETCH ( obj ) ;
2011-10-27 01:57:58 +02:00
2014-06-05 01:15:33 +02:00
* wp = ( Obj ) { obj , s - > elemsize , ti } ;
wp + + ;
nobj + + ;
continue_obj : ;
}
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
2012-12-22 02:15:33 +01:00
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2012-12-22 02:15:33 +01:00
}
2013-07-16 08:54:42 +02:00
static void
2014-06-05 01:15:33 +02:00
flushobjbuf ( Scanbuf * sbuf )
2013-07-16 08:54:42 +02:00
{
uintptr nobj , off ;
Obj * wp , obj ;
Workbuf * wbuf ;
2014-06-05 01:15:33 +02:00
Obj * objbuf ;
2013-07-16 08:54:42 +02:00
Obj * objbuf_end ;
2014-06-05 01:15:33 +02:00
wp = sbuf - > wp ;
wbuf = sbuf - > wbuf ;
nobj = sbuf - > nobj ;
2013-07-16 08:54:42 +02:00
2014-06-05 01:15:33 +02:00
objbuf = sbuf - > obj . begin ;
objbuf_end = sbuf - > obj . pos ;
sbuf - > obj . pos = sbuf - > obj . begin ;
2013-07-16 08:54:42 +02:00
while ( objbuf < objbuf_end ) {
obj = * objbuf + + ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
continue ;
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
}
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
2014-06-05 01:15:33 +02:00
sbuf - > wp = wp ;
sbuf - > wbuf = wbuf ;
sbuf - > nobj = nobj ;
2013-07-16 08:54:42 +02:00
}
2012-12-22 02:15:33 +01:00
// Program that scans the whole block and treats every block element as a potential pointer
static uintptr defaultProg [ 2 ] = { PtrSize , GC_DEFAULT_PTR } ;
2013-07-16 08:54:42 +02:00
// Hchan program
static uintptr chanProg [ 2 ] = { 0 , GC_CHAN } ;
2013-01-29 21:52:43 +01:00
// Local variables of a program fragment or loop
2016-08-30 23:07:47 +02:00
typedef struct GCFrame GCFrame ;
struct GCFrame {
2013-01-29 21:52:43 +01:00
uintptr count , elemsize , b ;
2014-09-04 00:56:09 +02:00
const uintptr * loop_or_ret ;
2013-01-29 21:52:43 +01:00
} ;
2013-07-16 08:54:42 +02:00
// Sanity check for the derived type info objti.
static void
checkptr ( void * obj , uintptr objti )
{
2014-09-04 00:56:09 +02:00
uintptr * pc1 , type , tisize , i , j , x ;
const uintptr * pc2 ;
2013-07-16 08:54:42 +02:00
byte * objstart ;
Type * t ;
MSpan * s ;
if ( ! Debug )
runtime_throw ( " checkptr is debug only " ) ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj < runtime_mheap . arena_start | | ( byte * ) obj > = runtime_mheap . arena_used )
2013-07-16 08:54:42 +02:00
return ;
type = runtime_gettype ( obj ) ;
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
if ( t = = nil )
return ;
x = ( uintptr ) obj > > PageShift ;
2014-01-10 00:16:56 +01:00
x - = ( uintptr ) ( runtime_mheap . arena_start ) > > PageShift ;
2013-11-06 20:49:01 +01:00
s = runtime_mheap . spans [ x ] ;
2013-07-16 08:54:42 +02:00
objstart = ( byte * ) ( ( uintptr ) s - > start < < PageShift ) ;
if ( s - > sizeclass ! = 0 ) {
i = ( ( byte * ) obj - objstart ) / s - > elemsize ;
objstart + = i * s - > elemsize ;
}
tisize = * ( uintptr * ) objti ;
// Sanity check for object size: it should fit into the memory block.
2013-11-06 20:49:01 +01:00
if ( ( byte * ) obj + tisize > objstart + s - > elemsize ) {
runtime_printf ( " object of type '%S' at %p/%p does not fit in block %p/%p \n " ,
* t - > string , obj , tisize , objstart , s - > elemsize ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
2013-11-06 20:49:01 +01:00
}
2013-07-16 08:54:42 +02:00
if ( obj ! = objstart )
return ;
// If obj points to the beginning of the memory block,
// check type info as well.
if ( t - > string = = nil | |
// Gob allocates unsafe pointers for indirection.
( runtime_strcmp ( ( const char * ) t - > string - > str , ( const char * ) " unsafe.Pointer " ) & &
// Runtime and gc think differently about closures.
runtime_strstr ( ( const char * ) t - > string - > str , ( const char * ) " struct { F uintptr " ) ! = ( const char * ) t - > string - > str ) ) {
pc1 = ( uintptr * ) objti ;
2014-09-04 00:56:09 +02:00
pc2 = ( const uintptr * ) t - > __gc ;
2013-07-16 08:54:42 +02:00
// A simple best-effort check until first GC_END.
for ( j = 1 ; pc1 [ j ] ! = GC_END & & pc2 [ j ] ! = GC_END ; j + + ) {
if ( pc1 [ j ] ! = pc2 [ j ] ) {
2014-07-19 10:53:52 +02:00
runtime_printf ( " invalid gc type info for '%s', type info %p [%d]=%p, block info %p [%d]=%p \n " ,
t - > string ? ( const int8 * ) t - > string - > str : ( const int8 * ) " ? " , pc1 , ( int32 ) j , pc1 [ j ] , pc2 , ( int32 ) j , pc2 [ j ] ) ;
2013-07-16 08:54:42 +02:00
runtime_throw ( " invalid gc type info " ) ;
}
}
}
}
2012-12-22 02:15:33 +01:00
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
static void
2014-06-07 00:37:27 +02:00
scanblock ( Workbuf * wbuf , bool keepworking )
2012-12-22 02:15:33 +01:00
{
byte * b , * arena_start , * arena_used ;
2014-09-04 00:56:09 +02:00
uintptr n , i , end_b , elemsize , size , ti , objti , count , type , nobj ;
uintptr precise_type , nominal_size ;
const uintptr * pc , * chan_ret ;
uintptr chancap ;
2012-12-22 02:15:33 +01:00
void * obj ;
2014-07-19 10:53:52 +02:00
const Type * t , * et ;
2013-01-29 21:52:43 +01:00
Slice * sliceptr ;
2014-07-19 10:53:52 +02:00
String * stringptr ;
2016-08-30 23:07:47 +02:00
GCFrame * stack_ptr , stack_top , stack [ GC_STACK_CAPACITY + 4 ] ;
2013-01-29 21:52:43 +01:00
BufferList * scanbuffers ;
2014-06-05 01:15:33 +02:00
Scanbuf sbuf ;
2013-01-29 21:52:43 +01:00
Eface * eface ;
Iface * iface ;
2013-07-16 08:54:42 +02:00
Hchan * chan ;
2014-09-04 00:56:09 +02:00
const ChanType * chantype ;
2014-06-07 00:37:27 +02:00
Obj * wp ;
2012-12-22 02:15:33 +01:00
2014-06-07 00:37:27 +02:00
if ( sizeof ( Workbuf ) % WorkbufSize ! = 0 )
2012-12-22 02:15:33 +01:00
runtime_throw ( " scanblock: size of Workbuf is suboptimal " ) ;
// Memory arena parameters.
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
arena_used = runtime_mheap . arena_used ;
2012-12-22 02:15:33 +01:00
2013-01-29 21:52:43 +01:00
stack_ptr = stack + nelem ( stack ) - 1 ;
2014-06-05 01:15:33 +02:00
2013-01-29 21:52:43 +01:00
precise_type = false ;
nominal_size = 0 ;
2014-06-07 00:37:27 +02:00
if ( wbuf ) {
nobj = wbuf - > nobj ;
wp = & wbuf - > obj [ nobj ] ;
} else {
nobj = 0 ;
wp = nil ;
}
2014-06-05 01:15:33 +02:00
// Initialize sbuf
scanbuffers = & bufferList [ runtime_m ( ) - > helpgc ] ;
sbuf . ptr . begin = sbuf . ptr . pos = & scanbuffers - > ptrtarget [ 0 ] ;
sbuf . ptr . end = sbuf . ptr . begin + nelem ( scanbuffers - > ptrtarget ) ;
sbuf . obj . begin = sbuf . obj . pos = & scanbuffers - > obj [ 0 ] ;
sbuf . obj . end = sbuf . obj . begin + nelem ( scanbuffers - > obj ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
sbuf . wbuf = wbuf ;
sbuf . wp = wp ;
sbuf . nobj = nobj ;
2013-07-16 08:54:42 +02:00
// (Silence the compiler)
chan = nil ;
chantype = nil ;
chan_ret = nil ;
2012-12-22 02:15:33 +01:00
goto next_block ;
for ( ; ; ) {
// Each iteration scans the block b of length n, queueing pointers in
// the work buffer.
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . nbytes , n ) ;
2014-06-05 01:15:33 +02:00
runtime_xadd64 ( & gcstats . obj . sum , sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
runtime_xadd64 ( & gcstats . obj . cnt , 1 ) ;
}
2014-09-04 00:56:09 +02:00
if ( ti ! = 0 ) {
2014-07-19 10:53:52 +02:00
if ( Debug > 1 ) {
runtime_printf ( " scanblock %p %D ti %p \n " , b , ( int64 ) n , ti ) ;
}
2013-01-29 21:52:43 +01:00
pc = ( uintptr * ) ( ti & ~ ( uintptr ) PC_BITS ) ;
precise_type = ( ti & PRECISE ) ;
stack_top . elemsize = pc [ 0 ] ;
if ( ! precise_type )
nominal_size = pc [ 0 ] ;
if ( ti & LOOP ) {
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . loop_or_ret = pc + 1 ;
} else {
stack_top . count = 1 ;
}
2013-07-16 08:54:42 +02:00
if ( Debug ) {
// Simple sanity check for provided type info ti:
// The declared size of the object must be not larger than the actual size
// (it can be smaller due to inferior pointers).
// It's difficult to make a comprehensive check due to inferior pointers,
// reflection, gob, etc.
if ( pc [ 0 ] > n ) {
runtime_printf ( " invalid gc type info: type info size %p, block size %p \n " , pc [ 0 ] , n ) ;
runtime_throw ( " invalid gc type info " ) ;
}
}
2014-09-04 00:56:09 +02:00
} else if ( UseSpanType ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . notype , 1 ) ;
2013-01-29 21:52:43 +01:00
type = runtime_gettype ( b ) ;
if ( type ! = 0 ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . obj . typelookup , 1 ) ;
2013-01-29 21:52:43 +01:00
t = ( Type * ) ( type & ~ ( uintptr ) ( PtrSize - 1 ) ) ;
switch ( type & ( PtrSize - 1 ) ) {
case TypeInfo_SingleObject :
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) t - > __gc ;
2013-01-29 21:52:43 +01:00
precise_type = true ; // type information about 'b' is precise
stack_top . count = 1 ;
stack_top . elemsize = pc [ 0 ] ;
break ;
case TypeInfo_Array :
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) t - > __gc ;
2013-01-29 21:52:43 +01:00
if ( pc [ 0 ] = = 0 )
goto next_block ;
precise_type = true ; // type information about 'b' is precise
stack_top . count = 0 ; // 0 means an infinite number of iterations
stack_top . elemsize = pc [ 0 ] ;
stack_top . loop_or_ret = pc + 1 ;
break ;
2013-07-16 08:54:42 +02:00
case TypeInfo_Chan :
chan = ( Hchan * ) b ;
2014-09-04 00:56:09 +02:00
chantype = ( const ChanType * ) t ;
2013-07-16 08:54:42 +02:00
chan_ret = nil ;
pc = chanProg ;
2013-01-29 21:52:43 +01:00
break ;
default :
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D type %p %S \n " , b , ( int64 ) n , type , * t - > string ) ;
2013-01-29 21:52:43 +01:00
runtime_throw ( " scanblock: invalid type " ) ;
return ;
}
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D type %p %S pc=%p \n " , b , ( int64 ) n , type , * t - > string , pc ) ;
2013-01-29 21:52:43 +01:00
} else {
pc = defaultProg ;
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D unknown type \n " , b , ( int64 ) n ) ;
2013-01-29 21:52:43 +01:00
}
} else {
pc = defaultProg ;
2014-07-19 10:53:52 +02:00
if ( Debug > 1 )
runtime_printf ( " scanblock %p %D no span types \n " , b , ( int64 ) n ) ;
2013-01-29 21:52:43 +01:00
}
2012-12-22 02:15:33 +01:00
2013-07-16 08:54:42 +02:00
if ( IgnorePreciseGC )
pc = defaultProg ;
2012-12-22 02:15:33 +01:00
pc + + ;
stack_top . b = ( uintptr ) b ;
end_b = ( uintptr ) b + n - PtrSize ;
2013-01-29 21:52:43 +01:00
for ( ; ; ) {
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . instr [ pc [ 0 ] ] , 1 ) ;
2013-01-29 21:52:43 +01:00
obj = nil ;
objti = 0 ;
2012-12-22 02:15:33 +01:00
switch ( pc [ 0 ] ) {
2013-01-29 21:52:43 +01:00
case GC_PTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
objti = pc [ 2 ] ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_ptr @%p: %p ti=%p \n " , stack_top . b + pc [ 1 ] , obj , objti ) ;
2013-01-29 21:52:43 +01:00
pc + = 3 ;
2013-07-16 08:54:42 +02:00
if ( Debug )
checkptr ( obj , objti ) ;
2013-01-29 21:52:43 +01:00
break ;
case GC_SLICE :
sliceptr = ( Slice * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_slice @%p: %p/%D/%D \n " , sliceptr , sliceptr - > array , ( int64 ) sliceptr - > __count , ( int64 ) sliceptr - > cap ) ;
2013-01-29 21:52:43 +01:00
if ( sliceptr - > cap ! = 0 ) {
obj = sliceptr - > array ;
2013-07-16 08:54:42 +02:00
// Can't use slice element type for scanning,
// because if it points to an array embedded
// in the beginning of a struct,
// we will scan the whole struct as the slice.
// So just obtain type info from heap.
2013-01-29 21:52:43 +01:00
}
pc + = 3 ;
break ;
case GC_APTR :
obj = * ( void * * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_aptr @%p: %p \n " , stack_top . b + pc [ 1 ] , obj ) ;
2013-01-29 21:52:43 +01:00
pc + = 2 ;
break ;
case GC_STRING :
2014-07-19 10:53:52 +02:00
stringptr = ( String * ) ( stack_top . b + pc [ 1 ] ) ;
if ( Debug > 2 )
runtime_printf ( " gc_string @%p: %p/%D \n " , stack_top . b + pc [ 1 ] , stringptr - > str , ( int64 ) stringptr - > len ) ;
if ( stringptr - > len ! = 0 )
markonly ( stringptr - > str ) ;
2013-01-29 21:52:43 +01:00
pc + = 2 ;
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
case GC_EFACE :
eface = ( Eface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_eface @%p: %p %p \n " , stack_top . b + pc [ 1 ] , eface - > __type_descriptor , eface - > __object ) ;
2014-06-07 00:37:27 +02:00
if ( eface - > __type_descriptor = = nil )
2013-07-16 08:54:42 +02:00
continue ;
// eface->type
2014-06-07 00:37:27 +02:00
t = eface - > __type_descriptor ;
2013-07-16 08:54:42 +02:00
if ( ( const byte * ) t > = arena_start & & ( const byte * ) t < arena_used ) {
union { const Type * tc ; Type * tr ; } u ;
u . tc = t ;
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { u . tr , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
// eface->__object
if ( ( byte * ) eface - > __object > = arena_start & & ( byte * ) eface - > __object < arena_used ) {
2014-09-04 00:56:09 +02:00
if ( __go_is_pointer_type ( t ) ) {
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( t - > __code & kindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = eface - > __object ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( t - > __code & kindMask ) = = kindPtr ) {
2014-07-19 10:53:52 +02:00
// Only use type information if it is a pointer-containing type.
// This matches the GC programs written by cmd/gc/reflect.c's
// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
et = ( ( const PtrType * ) t ) - > elem ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ! ( et - > __code & kindNoPointers ) )
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) ( ( const PtrType * ) t ) - > elem - > __gc ;
2014-07-19 10:53:52 +02:00
}
2013-01-29 21:52:43 +01:00
} else {
obj = eface - > __object ;
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) t - > __gc ;
2013-01-29 21:52:43 +01:00
}
}
break ;
case GC_IFACE :
iface = ( Iface * ) ( stack_top . b + pc [ 1 ] ) ;
pc + = 2 ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_iface @%p: %p/%p %p \n " , stack_top . b + pc [ 1 ] , iface - > __methods [ 0 ] , nil , iface - > __object ) ;
2013-01-29 21:52:43 +01:00
if ( iface - > tab = = nil )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
// iface->tab
if ( ( byte * ) iface - > tab > = arena_start & & ( byte * ) iface - > tab < arena_used ) {
2014-09-04 00:56:09 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { iface - > tab , 0 } ;
2014-06-05 01:15:33 +02:00
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
// iface->data
if ( ( byte * ) iface - > __object > = arena_start & & ( byte * ) iface - > __object < arena_used ) {
2014-09-04 00:56:09 +02:00
t = ( const Type * ) iface - > tab [ 0 ] ;
if ( __go_is_pointer_type ( t ) ) {
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( t - > __code & kindNoPointers ) )
2013-07-16 08:54:42 +02:00
continue ;
2013-01-29 21:52:43 +01:00
obj = iface - > __object ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( t - > __code & kindMask ) = = kindPtr ) {
2014-07-19 10:53:52 +02:00
// Only use type information if it is a pointer-containing type.
// This matches the GC programs written by cmd/gc/reflect.c's
// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
et = ( ( const PtrType * ) t ) - > elem ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ! ( et - > __code & kindNoPointers ) )
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) ( ( const PtrType * ) t ) - > elem - > __gc ;
2014-07-19 10:53:52 +02:00
}
2013-01-29 21:52:43 +01:00
} else {
obj = iface - > __object ;
2014-09-04 00:56:09 +02:00
objti = ( uintptr ) t - > __gc ;
2013-01-29 21:52:43 +01:00
}
}
break ;
2012-12-22 02:15:33 +01:00
case GC_DEFAULT_PTR :
2013-07-16 08:54:42 +02:00
while ( stack_top . b < = end_b ) {
obj = * ( byte * * ) stack_top . b ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_default_ptr @%p: %p \n " , stack_top . b , obj ) ;
2012-12-22 02:15:33 +01:00
stack_top . b + = PtrSize ;
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , 0 } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
}
}
2013-01-29 21:52:43 +01:00
goto next_block ;
case GC_END :
if ( - - stack_top . count ! = 0 ) {
// Next iteration of a loop if possible.
2013-07-16 08:54:42 +02:00
stack_top . b + = stack_top . elemsize ;
if ( stack_top . b + stack_top . elemsize < = end_b + PtrSize ) {
2013-01-29 21:52:43 +01:00
pc = stack_top . loop_or_ret ;
continue ;
}
i = stack_top . b ;
} else {
// Stack pop if possible.
if ( stack_ptr + 1 < stack + nelem ( stack ) ) {
pc = stack_top . loop_or_ret ;
stack_top = * ( + + stack_ptr ) ;
continue ;
}
i = ( uintptr ) b + nominal_size ;
}
if ( ! precise_type ) {
// Quickly scan [b+i,b+n) for possible pointers.
for ( ; i < = end_b ; i + = PtrSize ) {
if ( * ( byte * * ) i ! = nil ) {
// Found a value that may be a pointer.
// Do a rescan of the entire block.
2014-06-05 01:15:33 +02:00
enqueue ( ( Obj ) { b , n , 0 } , & sbuf . wbuf , & sbuf . wp , & sbuf . nobj ) ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_xadd64 ( & gcstats . rescan , 1 ) ;
runtime_xadd64 ( & gcstats . rescanbytes , n ) ;
}
2013-01-29 21:52:43 +01:00
break ;
}
}
}
goto next_block ;
case GC_ARRAY_START :
i = stack_top . b + pc [ 1 ] ;
count = pc [ 2 ] ;
elemsize = pc [ 3 ] ;
pc + = 4 ;
// Stack push.
* stack_ptr - - = stack_top ;
2016-08-30 23:07:47 +02:00
stack_top = ( GCFrame ) { count , elemsize , i , pc } ;
2013-01-29 21:52:43 +01:00
continue ;
case GC_ARRAY_NEXT :
if ( - - stack_top . count ! = 0 ) {
stack_top . b + = stack_top . elemsize ;
pc = stack_top . loop_or_ret ;
} else {
// Stack pop.
stack_top = * ( + + stack_ptr ) ;
pc + = 1 ;
}
continue ;
case GC_CALL :
// Stack push.
* stack_ptr - - = stack_top ;
2016-08-30 23:07:47 +02:00
stack_top = ( GCFrame ) { 1 , 0 , stack_top . b + pc [ 1 ] , pc + 3 /*return address*/ } ;
2014-09-04 00:56:09 +02:00
pc = ( const uintptr * ) ( ( const byte * ) pc + * ( const int32 * ) ( pc + 2 ) ) ; // target of the CALL instruction
2013-01-29 21:52:43 +01:00
continue ;
case GC_REGION :
obj = ( void * ) ( stack_top . b + pc [ 1 ] ) ;
2013-07-16 08:54:42 +02:00
size = pc [ 2 ] ;
objti = pc [ 3 ] ;
2013-01-29 21:52:43 +01:00
pc + = 4 ;
2013-07-16 08:54:42 +02:00
2014-07-19 10:53:52 +02:00
if ( Debug > 2 )
runtime_printf ( " gc_region @%p: %D %p \n " , stack_top . b + pc [ 1 ] , ( int64 ) size , objti ) ;
2014-06-05 01:15:33 +02:00
* sbuf . obj . pos + + = ( Obj ) { obj , size , objti } ;
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
continue ;
case GC_CHAN_PTR :
chan = * ( Hchan * * ) ( stack_top . b + pc [ 1 ] ) ;
2014-07-19 10:53:52 +02:00
if ( Debug > 2 & & chan ! = nil )
runtime_printf ( " gc_chan_ptr @%p: %p/%D/%D %p \n " , stack_top . b + pc [ 1 ] , chan , ( int64 ) chan - > qcount , ( int64 ) chan - > dataqsiz , pc [ 2 ] ) ;
2013-07-16 08:54:42 +02:00
if ( chan = = nil ) {
pc + = 3 ;
continue ;
}
if ( markonly ( chan ) ) {
chantype = ( ChanType * ) pc [ 2 ] ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ! ( chantype - > elem - > __code & kindNoPointers ) ) {
2013-07-16 08:54:42 +02:00
// Start chanProg.
chan_ret = pc + 3 ;
pc = chanProg + 1 ;
continue ;
}
}
pc + = 3 ;
continue ;
case GC_CHAN :
// There are no heap pointers in struct Hchan,
// so we can ignore the leading sizeof(Hchan) bytes.
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ! ( chantype - > elem - > __code & kindNoPointers ) ) {
2013-07-16 08:54:42 +02:00
// Channel's buffer follows Hchan immediately in memory.
// Size of buffer (cap(c)) is second int in the chan struct.
chancap = ( ( uintgo * ) chan ) [ 1 ] ;
if ( chancap > 0 ) {
// TODO(atom): split into two chunks so that only the
// in-use part of the circular buffer is scanned.
// (Channel routines zero the unused part, so the current
// code does not lead to leaks, it's just a little inefficient.)
2014-09-04 00:56:09 +02:00
* sbuf . obj . pos + + = ( Obj ) { ( byte * ) chan + runtime_Hchansize , chancap * chantype - > elem - > __size ,
( uintptr ) chantype - > elem - > __gc | PRECISE | LOOP } ;
2014-06-05 01:15:33 +02:00
if ( sbuf . obj . pos = = sbuf . obj . end )
flushobjbuf ( & sbuf ) ;
2013-07-16 08:54:42 +02:00
}
}
if ( chan_ret = = nil )
goto next_block ;
pc = chan_ret ;
continue ;
2012-12-22 02:15:33 +01:00
default :
2014-07-12 02:01:09 +02:00
runtime_printf ( " runtime: invalid GC instruction %p at %p \n " , pc [ 0 ] , pc ) ;
2012-12-22 02:15:33 +01:00
runtime_throw ( " scanblock: invalid GC instruction " ) ;
return ;
}
2013-01-29 21:52:43 +01:00
if ( ( byte * ) obj > = arena_start & & ( byte * ) obj < arena_used ) {
2014-06-05 01:15:33 +02:00
* sbuf . ptr . pos + + = ( PtrTarget ) { obj , objti } ;
if ( sbuf . ptr . pos = = sbuf . ptr . end )
flushptrbuf ( & sbuf ) ;
2013-01-29 21:52:43 +01:00
}
}
2012-12-22 02:15:33 +01:00
next_block :
2011-03-17 00:05:44 +01:00
// Done scanning [b, b+n). Prepare for the next iteration of
2013-01-29 21:52:43 +01:00
// the loop by setting b, n, ti to the parameters for the next block.
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
flushptrbuf ( & sbuf ) ;
flushobjbuf ( & sbuf ) ;
2012-12-22 02:15:33 +01:00
2014-06-05 01:15:33 +02:00
if ( sbuf . nobj = = 0 ) {
2012-12-22 02:15:33 +01:00
if ( ! keepworking ) {
2014-06-05 01:15:33 +02:00
if ( sbuf . wbuf )
putempty ( sbuf . wbuf ) ;
return ;
2012-12-22 02:15:33 +01:00
}
// Emptied our buffer: refill.
2014-06-05 01:15:33 +02:00
sbuf . wbuf = getfull ( sbuf . wbuf ) ;
if ( sbuf . wbuf = = nil )
return ;
sbuf . nobj = sbuf . wbuf - > nobj ;
sbuf . wp = sbuf . wbuf - > obj + sbuf . wbuf - > nobj ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
2012-12-22 02:15:33 +01:00
// Fetch b from the work buffer.
2014-06-05 01:15:33 +02:00
- - sbuf . wp ;
b = sbuf . wp - > p ;
n = sbuf . wp - > n ;
ti = sbuf . wp - > ti ;
sbuf . nobj - - ;
2011-03-17 00:05:44 +01:00
}
}
2014-06-07 00:37:27 +02:00
static struct root_list * roots ;
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
void
__go_register_gc_roots ( struct root_list * r )
{
// FIXME: This needs locking if multiple goroutines can call
// dlopen simultaneously.
r - > next = roots ;
roots = r ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
2012-12-22 02:15:33 +01:00
// Append obj to the work buffer.
// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
static void
enqueue ( Obj obj , Workbuf * * _wbuf , Obj * * _wp , uintptr * _nobj )
{
uintptr nobj , off ;
Obj * wp ;
Workbuf * wbuf ;
if ( Debug > 1 )
runtime_printf ( " append obj(%p %D %p) \n " , obj . p , ( int64 ) obj . n , obj . ti ) ;
// Align obj.b to a word boundary.
off = ( uintptr ) obj . p & ( PtrSize - 1 ) ;
if ( off ! = 0 ) {
obj . p + = PtrSize - off ;
obj . n - = PtrSize - off ;
obj . ti = 0 ;
}
if ( obj . p = = nil | | obj . n = = 0 )
return ;
// Load work buffer state
wp = * _wp ;
wbuf = * _wbuf ;
nobj = * _nobj ;
// If another proc wants a pointer, give it some.
if ( work . nwait > 0 & & nobj > handoffThreshold & & work . full = = 0 ) {
wbuf - > nobj = nobj ;
wbuf = handoff ( wbuf ) ;
nobj = wbuf - > nobj ;
wp = wbuf - > obj + nobj ;
}
// If buffer is full, get a new one.
if ( wbuf = = nil | | nobj > = nelem ( wbuf - > obj ) ) {
if ( wbuf ! = nil )
wbuf - > nobj = nobj ;
wbuf = getempty ( wbuf ) ;
wp = wbuf - > obj ;
nobj = 0 ;
}
* wp = obj ;
wp + + ;
nobj + + ;
// Save work buffer state
* _wp = wp ;
* _wbuf = wbuf ;
* _nobj = nobj ;
}
2014-06-07 00:37:27 +02:00
static void
enqueue1 ( Workbuf * * wbufp , Obj obj )
{
Workbuf * wbuf ;
wbuf = * wbufp ;
if ( wbuf - > nobj > = nelem ( wbuf - > obj ) )
* wbufp = wbuf = getempty ( wbuf ) ;
wbuf - > obj [ wbuf - > nobj + + ] = obj ;
}
2012-10-23 06:31:11 +02:00
static void
markroot ( ParFor * desc , uint32 i )
{
2012-12-22 02:15:33 +01:00
Workbuf * wbuf ;
2014-06-07 00:37:27 +02:00
FinBlock * fb ;
MHeap * h ;
MSpan * * allspans , * s ;
uint32 spanidx , sg ;
G * gp ;
void * p ;
2012-12-22 02:15:33 +01:00
2012-10-23 06:31:11 +02:00
USED ( & desc ) ;
2014-06-07 00:37:27 +02:00
wbuf = getempty ( nil ) ;
2014-07-19 10:53:52 +02:00
// Note: if you add a case here, please also update heapdump.c:dumproots.
2014-06-07 00:37:27 +02:00
switch ( i ) {
case RootData :
// For gccgo this is both data and bss.
{
struct root_list * pl ;
for ( pl = roots ; pl ! = nil ; pl = pl - > next ) {
struct root * pr = & pl - > roots [ 0 ] ;
while ( 1 ) {
void * decl = pr - > decl ;
if ( decl = = nil )
break ;
enqueue1 ( & wbuf , ( Obj ) { decl , pr - > size , 0 } ) ;
pr + + ;
}
}
}
break ;
case RootBss :
// For gccgo we use this for all the other global roots.
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_m0 , sizeof runtime_m0 , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_g0 , sizeof runtime_g0 , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allg , sizeof runtime_allg , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allm , sizeof runtime_allm , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & runtime_allp , sizeof runtime_allp , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) & work , sizeof work , 0 } ) ;
runtime_proc_scan ( & wbuf , enqueue1 ) ;
runtime_MProf_Mark ( & wbuf , enqueue1 ) ;
runtime_time_scan ( & wbuf , enqueue1 ) ;
runtime_netpoll_scan ( & wbuf , enqueue1 ) ;
break ;
case RootFinalizers :
for ( fb = allfin ; fb ; fb = fb - > alllink )
enqueue1 ( & wbuf , ( Obj ) { ( byte * ) fb - > fin , fb - > cnt * sizeof ( fb - > fin [ 0 ] ) , 0 } ) ;
break ;
case RootSpanTypes :
// mark span types and MSpan.specials (to walk spans only once)
h = & runtime_mheap ;
sg = h - > sweepgen ;
allspans = h - > allspans ;
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
Special * sp ;
SpecialFinalizer * spf ;
s = allspans [ spanidx ] ;
if ( s - > sweepgen ! = sg ) {
runtime_printf ( " sweep %d %d \n " , s - > sweepgen , sg ) ;
runtime_throw ( " gc: unswept span " ) ;
}
if ( s - > state ! = MSpanInUse )
continue ;
// The garbage collector ignores type pointers stored in MSpan.types:
// - Compiler-generated types are stored outside of heap.
// - The reflect package has runtime-generated types cached in its data structures.
// The garbage collector relies on finding the references via that cache.
if ( s - > types . compression = = MTypes_Words | | s - > types . compression = = MTypes_Bytes )
markonly ( ( byte * ) s - > types . data ) ;
for ( sp = s - > specials ; sp ! = nil ; sp = sp - > next ) {
if ( sp - > kind ! = KindSpecialFinalizer )
continue ;
// don't mark finalized object, but scan it so we
// retain everything it points to.
spf = ( SpecialFinalizer * ) sp ;
// A finalizer can be set for an inner byte of an object, find object beginning.
p = ( void * ) ( ( s - > start < < PageShift ) + spf - > offset / s - > elemsize * s - > elemsize ) ;
enqueue1 ( & wbuf , ( Obj ) { p , s - > elemsize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > fn , PtrSize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > ft , PtrSize , 0 } ) ;
enqueue1 ( & wbuf , ( Obj ) { ( void * ) & spf - > ot , PtrSize , 0 } ) ;
}
}
break ;
case RootFlushCaches :
flushallmcaches ( ) ;
break ;
default :
// the rest is scanning goroutine stacks
if ( i - RootCount > = runtime_allglen )
runtime_throw ( " markroot: bad index " ) ;
gp = runtime_allg [ i - RootCount ] ;
// remember when we've first observed the G blocked
// needed only to output in traceback
2016-08-30 23:07:47 +02:00
if ( ( gp - > atomicstatus = = _Gwaiting | | gp - > atomicstatus = = _Gsyscall ) & & gp - > waitsince = = 0 )
2014-06-07 00:37:27 +02:00
gp - > waitsince = work . tstart ;
addstackroots ( gp , & wbuf ) ;
break ;
}
if ( wbuf )
scanblock ( wbuf , false ) ;
2012-10-23 06:31:11 +02:00
}
2015-10-31 01:59:47 +01:00
static const FuncVal markroot_funcval = { ( void * ) markroot } ;
2011-03-17 00:05:44 +01:00
// Get an empty work buffer off the work.empty list,
// allocating new buffers as needed.
static Workbuf *
getempty ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
runtime_lfstackpush ( & work . full , & b - > node ) ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
b = ( Workbuf * ) runtime_lfstackpop ( & work . wempty ) ;
2012-10-23 06:31:11 +02:00
if ( b = = nil ) {
// Need to allocate.
runtime_lock ( & work ) ;
if ( work . nchunk < sizeof * b ) {
work . nchunk = 1 < < 20 ;
2013-11-06 20:49:01 +01:00
work . chunk = runtime_SysAlloc ( work . nchunk , & mstats . gc_sys ) ;
2013-07-16 08:54:42 +02:00
if ( work . chunk = = nil )
runtime_throw ( " runtime: cannot allocate memory " ) ;
2011-10-27 01:57:58 +02:00
}
2012-10-23 06:31:11 +02:00
b = ( Workbuf * ) work . chunk ;
work . chunk + = sizeof * b ;
work . nchunk - = sizeof * b ;
runtime_unlock ( & work ) ;
2011-03-17 00:05:44 +01:00
}
2011-10-27 01:57:58 +02:00
b - > nobj = 0 ;
2011-03-17 00:05:44 +01:00
return b ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
static void
putempty ( Workbuf * b )
{
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . putempty , 1 ) ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
runtime_lfstackpush ( & work . wempty , & b - > node ) ;
2011-10-27 01:57:58 +02:00
}
2011-03-17 00:05:44 +01:00
// Get a full work buffer off the work.full list, or return nil.
static Workbuf *
getfull ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 i ;
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_xadd64 ( & gcstats . getfull , 1 ) ;
2012-10-23 06:31:11 +02:00
if ( b ! = nil )
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
runtime_lfstackpush ( & work . wempty , & b - > node ) ;
2012-10-23 06:31:11 +02:00
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil | | work . nproc = = 1 )
2011-10-27 01:57:58 +02:00
return b ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
runtime_xadd ( & work . nwait , + 1 ) ;
for ( i = 0 ; ; i + + ) {
2012-10-23 06:31:11 +02:00
if ( work . full ! = 0 ) {
runtime_xadd ( & work . nwait , - 1 ) ;
b = ( Workbuf * ) runtime_lfstackpop ( & work . full ) ;
if ( b ! = nil )
return b ;
runtime_xadd ( & work . nwait , + 1 ) ;
2011-10-27 01:57:58 +02:00
}
if ( work . nwait = = work . nproc )
return nil ;
2012-10-23 06:31:11 +02:00
if ( i < 10 ) {
m - > gcstats . nprocyield + + ;
2011-10-27 01:57:58 +02:00
runtime_procyield ( 20 ) ;
2012-10-23 06:31:11 +02:00
} else if ( i < 20 ) {
m - > gcstats . nosyield + + ;
2011-10-27 01:57:58 +02:00
runtime_osyield ( ) ;
2012-10-23 06:31:11 +02:00
} else {
m - > gcstats . nsleep + + ;
2011-10-27 01:57:58 +02:00
runtime_usleep ( 100 ) ;
2012-10-23 06:31:11 +02:00
}
2011-03-17 00:05:44 +01:00
}
}
2011-10-27 01:57:58 +02:00
static Workbuf *
handoff ( Workbuf * b )
{
2012-10-23 06:31:11 +02:00
M * m ;
2011-10-27 01:57:58 +02:00
int32 n ;
Workbuf * b1 ;
2012-10-23 06:31:11 +02:00
m = runtime_m ( ) ;
2011-10-27 01:57:58 +02:00
// Make new buffer with half of b's pointers.
b1 = getempty ( nil ) ;
n = b - > nobj / 2 ;
b - > nobj - = n ;
b1 - > nobj = n ;
runtime_memmove ( b1 - > obj , b - > obj + b - > nobj , n * sizeof b1 - > obj [ 0 ] ) ;
2012-10-23 06:31:11 +02:00
m - > gcstats . nhandoff + + ;
m - > gcstats . nhandoffcnt + = n ;
2011-10-27 01:57:58 +02:00
// Put b on full list - let first half of b get stolen.
2012-10-23 06:31:11 +02:00
runtime_lfstackpush ( & work . full , & b - > node ) ;
2011-10-27 01:57:58 +02:00
return b1 ;
}
2011-11-28 06:45:49 +01:00
static void
2014-06-07 00:37:27 +02:00
addstackroots ( G * gp , Workbuf * * wbufp )
2012-10-23 06:31:11 +02:00
{
2016-08-30 23:07:47 +02:00
switch ( gp - > atomicstatus ) {
2014-06-07 00:37:27 +02:00
default :
2016-08-30 23:07:47 +02:00
runtime_printf ( " unexpected G.status %d (goroutine %p %D) \n " , gp - > atomicstatus , gp , gp - > goid ) ;
2014-06-07 00:37:27 +02:00
runtime_throw ( " mark - bad status " ) ;
2016-08-30 23:07:47 +02:00
case _Gdead :
2014-06-07 00:37:27 +02:00
return ;
2016-08-30 23:07:47 +02:00
case _Grunning :
2014-06-07 00:37:27 +02:00
runtime_throw ( " mark - world not stopped " ) ;
2016-08-30 23:07:47 +02:00
case _Grunnable :
case _Gsyscall :
case _Gwaiting :
2014-06-07 00:37:27 +02:00
break ;
2012-10-23 06:31:11 +02:00
}
2011-11-28 06:45:49 +01:00
# ifdef USING_SPLIT_STACK
M * mp ;
void * sp ;
size_t spsize ;
void * next_segment ;
void * next_sp ;
void * initial_sp ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
sp = __splitstack_find ( nil , nil , & spsize , & next_segment ,
& next_sp , & initial_sp ) ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
// The exception is that if the goroutine is about to enter or might
// have just exited a system call, it may be executing code such
// as schedlock and may have needed to start a new stack segment.
// Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot.
if ( gp - > gcstack ! = nil ) {
sp = gp - > gcstack ;
2016-08-30 23:07:47 +02:00
spsize = gp - > gcstacksize ;
next_segment = gp - > gcnextsegment ;
next_sp = gp - > gcnextsp ;
initial_sp = gp - > gcinitialsp ;
2011-11-28 06:45:49 +01:00
} else {
2016-08-30 23:07:47 +02:00
sp = __splitstack_find_context ( & gp - > stackcontext [ 0 ] ,
2011-11-28 06:45:49 +01:00
& spsize , & next_segment ,
& next_sp , & initial_sp ) ;
}
}
if ( sp ! = nil ) {
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
while ( ( sp = __splitstack_find ( next_segment , next_sp ,
& spsize , & next_segment ,
& next_sp , & initial_sp ) ) ! = nil )
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { sp , spsize , 0 } ) ;
2011-11-28 06:45:49 +01:00
}
# else
M * mp ;
byte * bottom ;
byte * top ;
if ( gp = = runtime_g ( ) ) {
// Scanning our own stack.
bottom = ( byte * ) & gp ;
} else if ( ( mp = gp - > m ) ! = nil & & mp - > helpgc ) {
// gchelper's stack is in active use and has no interesting pointers.
return ;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
2016-08-30 23:07:47 +02:00
bottom = ( byte * ) gp - > gcnextsp ;
2011-11-28 06:45:49 +01:00
if ( bottom = = nil )
return ;
}
2016-08-30 23:07:47 +02:00
top = ( byte * ) gp - > gcinitialsp + gp - > gcstacksize ;
2011-11-28 06:45:49 +01:00
if ( top > bottom )
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { bottom , top - bottom , 0 } ) ;
2011-11-28 06:45:49 +01:00
else
2014-06-07 00:37:27 +02:00
enqueue1 ( wbufp , ( Obj ) { top , bottom - top , 0 } ) ;
2011-11-28 06:45:49 +01:00
# endif
}
2010-12-03 05:34:57 +01:00
void
2014-06-07 00:37:27 +02:00
runtime_queuefinalizer ( void * p , FuncVal * fn , const FuncType * ft , const PtrType * ot )
2011-10-27 01:57:58 +02:00
{
FinBlock * block ;
Finalizer * f ;
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
if ( finq = = nil | | finq - > cnt = = finq - > cap ) {
if ( finc = = nil ) {
2014-06-07 00:37:27 +02:00
finc = runtime_persistentalloc ( FinBlockSize , 0 , & mstats . gc_sys ) ;
finc - > cap = ( FinBlockSize - sizeof ( FinBlock ) ) / sizeof ( Finalizer ) + 1 ;
2011-10-27 01:57:58 +02:00
finc - > alllink = allfin ;
allfin = finc ;
}
block = finc ;
finc = block - > next ;
block - > next = finq ;
finq = block ;
}
f = & finq - > fin [ finq - > cnt ] ;
finq - > cnt + + ;
f - > fn = fn ;
f - > ft = ft ;
2013-11-06 20:49:01 +01:00
f - > ot = ot ;
2011-10-27 01:57:58 +02:00
f - > arg = p ;
2014-07-19 10:53:52 +02:00
runtime_fingwake = true ;
runtime_unlock ( & finlock ) ;
}
void
runtime_iterate_finq ( void ( * callback ) ( FuncVal * , void * , const FuncType * , const PtrType * ) )
{
FinBlock * fb ;
Finalizer * f ;
int32 i ;
for ( fb = allfin ; fb ; fb = fb - > alllink ) {
for ( i = 0 ; i < fb - > cnt ; i + + ) {
f = & fb - > fin [ i ] ;
callback ( f - > fn , f - > arg , f - > ft , f - > ot ) ;
}
}
2014-06-07 00:37:27 +02:00
}
void
runtime_MSpan_EnsureSwept ( MSpan * s )
{
M * m = runtime_m ( ) ;
2014-07-19 10:53:52 +02:00
G * g = runtime_g ( ) ;
2014-06-07 00:37:27 +02:00
uint32 sg ;
2014-07-19 10:53:52 +02:00
// Caller must disable preemption.
// Otherwise when this function returns the span can become unswept again
// (if GC is triggered on another goroutine).
if ( m - > locks = = 0 & & m - > mallocing = = 0 & & g ! = m - > g0 )
runtime_throw ( " MSpan_EnsureSwept: m is not locked " ) ;
2014-06-07 00:37:27 +02:00
sg = runtime_mheap . sweepgen ;
if ( runtime_atomicload ( & s - > sweepgen ) = = sg )
return ;
if ( runtime_cas ( & s - > sweepgen , sg - 2 , sg - 1 ) ) {
runtime_MSpan_Sweep ( s ) ;
return ;
}
// unfortunate condition, and we don't have efficient means to wait
while ( runtime_atomicload ( & s - > sweepgen ) ! = sg )
runtime_osyield ( ) ;
2011-10-27 01:57:58 +02:00
}
// Sweep frees or collects finalizers for blocks not marked in the mark phase.
2011-03-17 00:05:44 +01:00
// It clears the mark bits in preparation for the next GC round.
2014-06-07 00:37:27 +02:00
// Returns true if the span was returned to heap.
bool
runtime_MSpan_Sweep ( MSpan * s )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2014-06-07 00:37:27 +02:00
int32 cl , n , npages , nfree ;
uintptr size , off , * bitp , shift , bits ;
uint32 sweepgen ;
2010-12-03 05:34:57 +01:00
byte * p ;
MCache * c ;
2011-10-27 01:57:58 +02:00
byte * arena_start ;
2012-10-23 06:31:11 +02:00
MLink head , * end ;
byte * type_data ;
byte compression ;
uintptr type_data_inc ;
2014-06-05 01:15:33 +02:00
MLink * x ;
2014-06-07 00:37:27 +02:00
Special * special , * * specialp , * y ;
bool res , sweepgenset ;
2011-10-27 01:57:58 +02:00
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2012-10-23 06:31:11 +02:00
2014-06-07 00:37:27 +02:00
// It's critical that we enter this function with preemption disabled,
// GC must not start while we are in the middle of this function.
if ( m - > locks = = 0 & & m - > mallocing = = 0 & & runtime_g ( ) ! = m - > g0 )
runtime_throw ( " MSpan_Sweep: m is not locked " ) ;
sweepgen = runtime_mheap . sweepgen ;
if ( s - > state ! = MSpanInUse | | s - > sweepgen ! = sweepgen - 1 ) {
runtime_printf ( " MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d \n " ,
s - > state , s - > sweepgen , sweepgen ) ;
runtime_throw ( " MSpan_Sweep: bad span state " ) ;
}
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-10-23 06:31:11 +02:00
cl = s - > sizeclass ;
size = s - > elemsize ;
if ( cl = = 0 ) {
n = 1 ;
} else {
// Chunk full of small blocks.
npages = runtime_class_to_allocnpages [ cl ] ;
n = ( npages < < PageShift ) / size ;
}
2014-06-07 00:37:27 +02:00
res = false ;
2012-10-23 06:31:11 +02:00
nfree = 0 ;
end = & head ;
c = m - > mcache ;
2014-06-07 00:37:27 +02:00
sweepgenset = false ;
2014-06-05 01:15:33 +02:00
// mark any free objects in this span so we don't collect them
for ( x = s - > freelist ; x ! = nil ; x = x - > next ) {
// This is markonly(x) but faster because we don't need
// atomic access and we're guaranteed to be pointing at
// the head of a valid object.
off = ( uintptr * ) x - ( uintptr * ) runtime_mheap . arena_start ;
bitp = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
* bitp | = bitMarked < < shift ;
}
2014-07-19 10:53:52 +02:00
2014-06-07 00:37:27 +02:00
// Unlink & free special records for any objects we're about to free.
specialp = & s - > specials ;
special = * specialp ;
while ( special ! = nil ) {
// A finalizer can be set for an inner byte of an object, find object beginning.
p = ( byte * ) ( s - > start < < PageShift ) + special - > offset / size * size ;
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
if ( ( bits & ( bitAllocated | bitMarked ) ) = = bitAllocated ) {
// Find the exact byte for which the special was setup
// (as opposed to object beginning).
p = ( byte * ) ( s - > start < < PageShift ) + special - > offset ;
// about to free object: splice out special record
y = special ;
special = special - > next ;
* specialp = special ;
if ( ! runtime_freespecial ( y , p , size , false ) ) {
// stop freeing of object if it has a finalizer
* bitp | = bitMarked < < shift ;
}
} else {
// object is still live: keep special record
specialp = & special - > next ;
special = * specialp ;
}
}
2012-10-23 06:31:11 +02:00
type_data = ( byte * ) s - > types . data ;
type_data_inc = sizeof ( uintptr ) ;
compression = s - > types . compression ;
switch ( compression ) {
case MTypes_Bytes :
type_data + = 8 * sizeof ( uintptr ) ;
type_data_inc = 1 ;
break ;
}
2011-10-27 01:57:58 +02:00
2012-10-23 06:31:11 +02:00
// Sweep through n objects of given size starting at p.
// This thread owns the span now, so it can manipulate
// the block bitmap without atomic operations.
2014-06-07 00:37:27 +02:00
p = ( byte * ) ( s - > start < < PageShift ) ;
2012-10-23 06:31:11 +02:00
for ( ; n > 0 ; n - - , p + = size , type_data + = type_data_inc ) {
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
2012-03-02 21:01:37 +01:00
2012-10-23 06:31:11 +02:00
if ( ( bits & bitAllocated ) = = 0 )
2011-03-17 00:05:44 +01:00
continue ;
2012-10-23 06:31:11 +02:00
if ( ( bits & bitMarked ) ! = 0 ) {
* bitp & = ~ ( bitMarked < < shift ) ;
continue ;
2010-12-03 05:34:57 +01:00
}
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
if ( runtime_debug . allocfreetrace )
runtime_tracefree ( p , size ) ;
// Clear mark and scan bits.
* bitp & = ~ ( ( bitScan | bitMarked ) < < shift ) ;
2011-03-17 00:05:44 +01:00
2012-10-23 06:31:11 +02:00
if ( cl = = 0 ) {
// Free large span.
runtime_unmarkspan ( p , 1 < < PageShift ) ;
2014-06-07 00:37:27 +02:00
s - > needzero = 1 ;
// important to set sweepgen before returning it to heap
runtime_atomicstore ( & s - > sweepgen , sweepgen ) ;
sweepgenset = true ;
2014-07-19 10:53:52 +02:00
// See note about SysFault vs SysFree in malloc.goc.
2014-06-05 01:15:33 +02:00
if ( runtime_debug . efence )
2014-07-19 10:53:52 +02:00
runtime_SysFault ( p , size ) ;
2014-06-05 01:15:33 +02:00
else
runtime_MHeap_Free ( & runtime_mheap , s , 1 ) ;
2013-11-06 20:49:01 +01:00
c - > local_nlargefree + + ;
c - > local_largefree + = size ;
2014-06-07 00:37:27 +02:00
runtime_xadd64 ( & mstats . next_gc , - ( uint64 ) ( size * ( gcpercent + 100 ) / 100 ) ) ;
res = true ;
2012-10-23 06:31:11 +02:00
} else {
// Free small object.
switch ( compression ) {
case MTypes_Words :
* ( uintptr * ) type_data = 0 ;
break ;
case MTypes_Bytes :
* ( byte * ) type_data = 0 ;
break ;
}
2014-06-07 00:37:27 +02:00
if ( size > 2 * sizeof ( uintptr ) )
2013-07-16 08:54:42 +02:00
( ( uintptr * ) p ) [ 1 ] = ( uintptr ) 0xdeaddeaddeaddeadll ; // mark as "needs to be zeroed"
2014-06-07 00:37:27 +02:00
else if ( size > sizeof ( uintptr ) )
( ( uintptr * ) p ) [ 1 ] = 0 ;
2012-10-23 06:31:11 +02:00
end - > next = ( MLink * ) p ;
end = ( MLink * ) p ;
nfree + + ;
2010-12-03 05:34:57 +01:00
}
}
2012-10-23 06:31:11 +02:00
2014-07-19 10:53:52 +02:00
// We need to set s->sweepgen = h->sweepgen only when all blocks are swept,
// because of the potential for a concurrent free/SetFinalizer.
// But we need to set it before we make the span available for allocation
// (return it to heap or mcentral), because allocation code assumes that a
// span is already swept if available for allocation.
if ( ! sweepgenset & & nfree = = 0 ) {
2014-06-07 00:37:27 +02:00
// The span must be in our exclusive ownership until we update sweepgen,
// check for potential races.
if ( s - > state ! = MSpanInUse | | s - > sweepgen ! = sweepgen - 1 ) {
runtime_printf ( " MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d \n " ,
s - > state , s - > sweepgen , sweepgen ) ;
runtime_throw ( " MSpan_Sweep: bad span state after sweep " ) ;
}
runtime_atomicstore ( & s - > sweepgen , sweepgen ) ;
}
2014-07-19 10:53:52 +02:00
if ( nfree > 0 ) {
2013-11-06 20:49:01 +01:00
c - > local_nsmallfree [ cl ] + = nfree ;
2012-10-23 06:31:11 +02:00
c - > local_cachealloc - = nfree * size ;
2014-06-07 00:37:27 +02:00
runtime_xadd64 ( & mstats . next_gc , - ( uint64 ) ( nfree * size * ( gcpercent + 100 ) / 100 ) ) ;
res = runtime_MCentral_FreeSpan ( & runtime_mheap . central [ cl ] , s , nfree , head . next , end ) ;
2014-07-19 10:53:52 +02:00
//MCentral_FreeSpan updates sweepgen
2014-06-07 00:37:27 +02:00
}
return res ;
}
// State of background sweep.
2014-09-04 00:56:09 +02:00
// Protected by gclock.
2014-06-07 00:37:27 +02:00
static struct
{
G * g ;
bool parked ;
MSpan * * spans ;
uint32 nspan ;
uint32 spanidx ;
} sweep ;
// background sweeping goroutine
static void
bgsweep ( void * dummy __attribute__ ( ( unused ) ) )
{
runtime_g ( ) - > issystem = 1 ;
for ( ; ; ) {
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 ) {
gcstats . nbgsweep + + ;
runtime_gosched ( ) ;
}
runtime_lock ( & gclock ) ;
2014-07-19 10:53:52 +02:00
if ( ! runtime_mheap . sweepdone ) {
// It's possible if GC has happened between sweepone has
// returned -1 and gclock lock.
runtime_unlock ( & gclock ) ;
continue ;
2014-06-07 00:37:27 +02:00
}
sweep . parked = true ;
2014-07-19 10:53:52 +02:00
runtime_g ( ) - > isbackground = true ;
2014-06-07 00:37:27 +02:00
runtime_parkunlock ( & gclock , " GC sweep wait " ) ;
2014-07-19 10:53:52 +02:00
runtime_g ( ) - > isbackground = false ;
2014-06-07 00:37:27 +02:00
}
}
// sweeps one span
// returns number of pages returned to heap, or -1 if there is nothing to sweep
uintptr
runtime_sweepone ( void )
{
M * m = runtime_m ( ) ;
MSpan * s ;
uint32 idx , sg ;
uintptr npages ;
// increment locks to ensure that the goroutine is not preempted
// in the middle of sweep thus leaving the span in an inconsistent state for next GC
m - > locks + + ;
sg = runtime_mheap . sweepgen ;
for ( ; ; ) {
idx = runtime_xadd ( & sweep . spanidx , 1 ) - 1 ;
if ( idx > = sweep . nspan ) {
runtime_mheap . sweepdone = true ;
m - > locks - - ;
return ( uintptr ) - 1 ;
}
s = sweep . spans [ idx ] ;
if ( s - > state ! = MSpanInUse ) {
s - > sweepgen = sg ;
continue ;
}
if ( s - > sweepgen ! = sg - 2 | | ! runtime_cas ( & s - > sweepgen , sg - 2 , sg - 1 ) )
continue ;
2014-07-19 10:53:52 +02:00
if ( s - > incache )
runtime_throw ( " sweep of incache span " ) ;
2014-06-07 00:37:27 +02:00
npages = s - > npages ;
if ( ! runtime_MSpan_Sweep ( s ) )
npages = 0 ;
m - > locks - - ;
return npages ;
2012-10-23 06:31:11 +02:00
}
2010-12-03 05:34:57 +01:00
}
2012-11-21 08:03:38 +01:00
static void
dumpspan ( uint32 idx )
{
int32 sizeclass , n , npages , i , column ;
uintptr size ;
byte * p ;
byte * arena_start ;
MSpan * s ;
2014-07-19 10:53:52 +02:00
bool allocated ;
2012-11-21 08:03:38 +01:00
2013-11-06 20:49:01 +01:00
s = runtime_mheap . allspans [ idx ] ;
2012-11-21 08:03:38 +01:00
if ( s - > state ! = MSpanInUse )
return ;
2013-11-06 20:49:01 +01:00
arena_start = runtime_mheap . arena_start ;
2012-11-21 08:03:38 +01:00
p = ( byte * ) ( s - > start < < PageShift ) ;
sizeclass = s - > sizeclass ;
size = s - > elemsize ;
if ( sizeclass = = 0 ) {
n = 1 ;
} else {
npages = runtime_class_to_allocnpages [ sizeclass ] ;
n = ( npages < < PageShift ) / size ;
}
runtime_printf ( " %p .. %p: \n " , p , p + n * size ) ;
column = 0 ;
for ( ; n > 0 ; n - - , p + = size ) {
uintptr off , * bitp , shift , bits ;
off = ( uintptr * ) p - ( uintptr * ) arena_start ;
bitp = ( uintptr * ) arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
bits = * bitp > > shift ;
allocated = ( ( bits & bitAllocated ) ! = 0 ) ;
for ( i = 0 ; ( uint32 ) i < size ; i + = sizeof ( void * ) ) {
if ( column = = 0 ) {
runtime_printf ( " \t " ) ;
}
if ( i = = 0 ) {
runtime_printf ( allocated ? " ( " : " [ " ) ;
runtime_printf ( " %p: " , p + i ) ;
} else {
runtime_printf ( " " ) ;
}
runtime_printf ( " %p " , * ( void * * ) ( p + i ) ) ;
if ( i + sizeof ( void * ) > = size ) {
runtime_printf ( allocated ? " ) " : " ] " ) ;
}
column + + ;
if ( column = = 8 ) {
runtime_printf ( " \n " ) ;
column = 0 ;
}
}
}
runtime_printf ( " \n " ) ;
}
// A debugging function to dump the contents of memory
void
runtime_memorydump ( void )
{
uint32 spanidx ;
2013-11-06 20:49:01 +01:00
for ( spanidx = 0 ; spanidx < runtime_mheap . nspan ; spanidx + + ) {
2012-11-21 08:03:38 +01:00
dumpspan ( spanidx ) ;
}
}
2012-12-13 00:13:29 +01:00
2011-10-27 01:57:58 +02:00
void
runtime_gchelper ( void )
{
2014-03-03 21:14:52 +01:00
uint32 nproc ;
2014-07-19 10:53:52 +02:00
runtime_m ( ) - > traceback = 2 ;
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
// parallel mark for over gc roots
runtime_parfordo ( work . markfor ) ;
2012-12-22 02:15:33 +01:00
2012-10-23 06:31:11 +02:00
// help other threads scan secondary blocks
2014-06-07 00:37:27 +02:00
scanblock ( nil , true ) ;
2011-10-27 01:57:58 +02:00
2013-07-16 08:54:42 +02:00
bufferList [ runtime_m ( ) - > helpgc ] . busy = 0 ;
2014-03-03 21:14:52 +01:00
nproc = work . nproc ; // work.nproc can change right after we increment work.ndone
if ( runtime_xadd ( & work . ndone , + 1 ) = = nproc - 1 )
2011-10-27 01:57:58 +02:00
runtime_notewakeup ( & work . alldone ) ;
2014-07-19 10:53:52 +02:00
runtime_m ( ) - > traceback = 0 ;
2011-10-27 01:57:58 +02:00
}
2011-11-28 06:45:49 +01:00
static void
2013-11-06 20:49:01 +01:00
cachestats ( void )
{
MCache * c ;
P * p , * * pp ;
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
runtime_purgecachedstats ( c ) ;
}
}
2014-06-07 00:37:27 +02:00
static void
flushallmcaches ( void )
{
P * p , * * pp ;
MCache * c ;
// Flush MCache's to MCentral.
for ( pp = runtime_allp ; ( p = * pp ) ! = nil ; pp + + ) {
c = p - > mcache ;
if ( c = = nil )
continue ;
runtime_MCache_ReleaseAll ( c ) ;
}
}
2014-07-19 10:53:52 +02:00
void
runtime_updatememstats ( GCStats * stats )
2011-11-28 06:45:49 +01:00
{
2012-12-22 02:15:33 +01:00
M * mp ;
2013-11-06 20:49:01 +01:00
MSpan * s ;
2011-11-28 06:45:49 +01:00
uint32 i ;
2013-11-06 20:49:01 +01:00
uint64 stacks_inuse , smallfree ;
2012-10-23 06:31:11 +02:00
uint64 * src , * dst ;
2011-11-28 06:45:49 +01:00
2012-10-23 06:31:11 +02:00
if ( stats )
runtime_memclr ( ( byte * ) stats , sizeof ( * stats ) ) ;
2011-11-28 06:45:49 +01:00
stacks_inuse = 0 ;
2012-12-22 02:15:33 +01:00
for ( mp = runtime_allm ; mp ; mp = mp - > alllink ) {
2013-07-16 08:54:42 +02:00
//stacks_inuse += mp->stackinuse*FixedStack;
2012-10-23 06:31:11 +02:00
if ( stats ) {
2012-12-22 02:15:33 +01:00
src = ( uint64 * ) & mp - > gcstats ;
2012-10-23 06:31:11 +02:00
dst = ( uint64 * ) stats ;
for ( i = 0 ; i < sizeof ( * stats ) / sizeof ( uint64 ) ; i + + )
dst [ i ] + = src [ i ] ;
2012-12-22 02:15:33 +01:00
runtime_memclr ( ( byte * ) & mp - > gcstats , sizeof ( mp - > gcstats ) ) ;
2012-10-23 06:31:11 +02:00
}
2013-07-16 08:54:42 +02:00
}
2013-11-06 20:49:01 +01:00
mstats . stacks_inuse = stacks_inuse ;
mstats . mcache_inuse = runtime_mheap . cachealloc . inuse ;
mstats . mspan_inuse = runtime_mheap . spanalloc . inuse ;
mstats . sys = mstats . heap_sys + mstats . stacks_sys + mstats . mspan_sys +
mstats . mcache_sys + mstats . buckhash_sys + mstats . gc_sys + mstats . other_sys ;
// Calculate memory allocator stats.
// During program execution we only count number of frees and amount of freed memory.
// Current number of alive object in the heap and amount of alive heap memory
// are calculated by scanning all spans.
// Total number of mallocs is calculated as number of frees plus number of alive objects.
// Similarly, total amount of allocated memory is calculated as amount of freed memory
// plus amount of alive heap memory.
mstats . alloc = 0 ;
mstats . total_alloc = 0 ;
mstats . nmalloc = 0 ;
mstats . nfree = 0 ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . by_size [ i ] . nmalloc = 0 ;
mstats . by_size [ i ] . nfree = 0 ;
}
// Flush MCache's to MCentral.
2014-06-07 00:37:27 +02:00
flushallmcaches ( ) ;
2013-11-06 20:49:01 +01:00
// Aggregate local stats.
cachestats ( ) ;
// Scan all spans and count number of alive objects.
for ( i = 0 ; i < runtime_mheap . nspan ; i + + ) {
s = runtime_mheap . allspans [ i ] ;
if ( s - > state ! = MSpanInUse )
continue ;
if ( s - > sizeclass = = 0 ) {
mstats . nmalloc + + ;
mstats . alloc + = s - > elemsize ;
} else {
mstats . nmalloc + = s - > ref ;
mstats . by_size [ s - > sizeclass ] . nmalloc + = s - > ref ;
mstats . alloc + = s - > ref * s - > elemsize ;
2011-11-28 06:45:49 +01:00
}
}
2013-11-06 20:49:01 +01:00
// Aggregate by size class.
smallfree = 0 ;
mstats . nfree = runtime_mheap . nlargefree ;
for ( i = 0 ; i < nelem ( mstats . by_size ) ; i + + ) {
mstats . nfree + = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nfree = runtime_mheap . nsmallfree [ i ] ;
mstats . by_size [ i ] . nmalloc + = runtime_mheap . nsmallfree [ i ] ;
smallfree + = runtime_mheap . nsmallfree [ i ] * runtime_class_to_size [ i ] ;
}
mstats . nmalloc + = mstats . nfree ;
// Calculate derived stats.
mstats . total_alloc = mstats . alloc + runtime_mheap . largefree + smallfree ;
mstats . heap_alloc = mstats . alloc ;
mstats . heap_objects = mstats . nmalloc - mstats . nfree ;
2011-11-28 06:45:49 +01:00
}
2012-12-13 00:13:29 +01:00
// Structure of arguments passed to function gc().
2013-11-06 20:49:01 +01:00
// This allows the arguments to be passed via runtime_mcall.
2012-12-13 00:13:29 +01:00
struct gc_args
{
2013-11-06 20:49:01 +01:00
int64 start_time ; // start time of GC in ns (just before stoptheworld)
2014-07-19 10:53:52 +02:00
bool eagersweep ;
2012-12-13 00:13:29 +01:00
} ;
static void gc ( struct gc_args * args ) ;
2013-11-06 20:49:01 +01:00
static void mgc ( G * gp ) ;
2012-12-13 00:13:29 +01:00
2013-07-16 08:54:42 +02:00
static int32
readgogc ( void )
{
2015-10-31 01:59:47 +01:00
String s ;
2013-07-16 08:54:42 +02:00
const byte * p ;
2015-10-31 01:59:47 +01:00
s = runtime_getenv ( " GOGC " ) ;
if ( s . len = = 0 )
2013-07-16 08:54:42 +02:00
return 100 ;
2015-10-31 01:59:47 +01:00
p = s . str ;
if ( s . len = = 3 & & runtime_strcmp ( ( const char * ) p , " off " ) = = 0 )
2013-07-16 08:54:42 +02:00
return - 1 ;
2015-10-31 01:59:47 +01:00
return runtime_atoi ( p , s . len ) ;
2013-07-16 08:54:42 +02:00
}
2014-07-19 10:53:52 +02:00
// force = 1 - do GC regardless of current heap usage
// force = 2 - go GC and eager sweep
2010-12-03 05:34:57 +01:00
void
2011-11-28 06:45:49 +01:00
runtime_gc ( int32 force )
2010-12-03 05:34:57 +01:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2013-11-06 20:49:01 +01:00
G * g ;
struct gc_args a ;
int32 i ;
2012-10-23 06:31:11 +02:00
// The atomic operations are not atomic if the uint64s
// are not aligned on uint64 boundaries. This has been
// a problem in the past.
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( ( ( uintptr ) & work . wempty ) & 7 ) ! = 0 )
2012-10-23 06:31:11 +02:00
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2013-07-16 08:54:42 +02:00
if ( ( ( ( uintptr ) & work . full ) & 7 ) ! = 0 )
runtime_throw ( " runtime: gc work buffer is misaligned " ) ;
2010-12-03 05:34:57 +01:00
2012-02-10 16:52:37 +01:00
// Make sure all registers are saved on stack so that
// scanstack sees them.
__builtin_unwind_init ( ) ;
2010-12-03 05:34:57 +01:00
// The gc is turned off (via enablegc) until
// the bootstrap has completed.
// Also, malloc gets called in the guts
// of a number of libraries that might be
// holding locks. To avoid priority inversion
// problems, don't bother trying to run gc
// while holding a lock. The next mallocgc
// without a lock will do the gc instead.
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2013-11-06 20:49:01 +01:00
if ( ! mstats . enablegc | | runtime_g ( ) = = m - > g0 | | m - > locks > 0 | | runtime_panicking )
2010-12-03 05:34:57 +01:00
return ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown ) { // first time through
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
runtime_unlock ( & runtime_mheap ) ;
2010-12-03 05:34:57 +01:00
}
if ( gcpercent < 0 )
return ;
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
2014-07-19 10:53:52 +02:00
if ( force = = 0 & & mstats . heap_alloc < mstats . next_gc ) {
2013-11-06 20:49:01 +01:00
// typically threads which lost the race to grab
// worldsema exit here when gc is done.
runtime_semrelease ( & runtime_worldsema ) ;
return ;
}
2012-12-13 00:13:29 +01:00
2013-11-06 20:49:01 +01:00
// Ok, we're doing it! Stop everybody else
a . start_time = runtime_nanotime ( ) ;
2014-07-19 10:53:52 +02:00
a . eagersweep = force > = 2 ;
2013-11-06 20:49:01 +01:00
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2014-06-07 00:37:27 +02:00
2014-06-05 01:15:33 +02:00
clearpools ( ) ;
2013-11-06 20:49:01 +01:00
// Run gc on the g0 stack. We do this so that the g stack
// we're currently running on will no longer change. Cuts
// the root set down a bit (g0 stacks are not scanned, and
// we don't need to scan gc's internal state). Also an
// enabler for copyable stacks.
for ( i = 0 ; i < ( runtime_debug . gctrace > 1 ? 2 : 1 ) ; i + + ) {
2014-07-19 10:53:52 +02:00
if ( i > 0 )
a . start_time = runtime_nanotime ( ) ;
2013-11-06 20:49:01 +01:00
// switch to g0, call gc(&a), then switch back
g = runtime_g ( ) ;
g - > param = & a ;
2016-08-30 23:07:47 +02:00
g - > atomicstatus = _Gwaiting ;
g - > waitreason = runtime_gostringnocopy ( ( const byte * ) " garbage collection " ) ;
2013-11-06 20:49:01 +01:00
runtime_mcall ( mgc ) ;
2014-08-16 00:16:55 +02:00
m = runtime_m ( ) ;
2012-12-13 00:13:29 +01:00
}
2013-11-06 20:49:01 +01:00
// all done
m - > gcing = 0 ;
m - > locks + + ;
runtime_semrelease ( & runtime_worldsema ) ;
runtime_starttheworld ( ) ;
m - > locks - - ;
// now that gc is done, kick off finalizer thread if needed
2014-06-07 00:37:27 +02:00
if ( ! ConcurrentSweep ) {
// give the queued finalizers, if any, a chance to run
runtime_gosched ( ) ;
} else {
// For gccgo, let other goroutines run.
runtime_gosched ( ) ;
2013-11-06 20:49:01 +01:00
}
}
static void
mgc ( G * gp )
{
gc ( gp - > param ) ;
gp - > param = nil ;
2016-08-30 23:07:47 +02:00
gp - > atomicstatus = _Grunning ;
2013-11-06 20:49:01 +01:00
runtime_gogo ( gp ) ;
2012-12-13 00:13:29 +01:00
}
static void
gc ( struct gc_args * args )
{
M * m ;
2013-01-29 21:52:43 +01:00
int64 t0 , t1 , t2 , t3 , t4 ;
2014-06-07 00:37:27 +02:00
uint64 heap0 , heap1 , obj , ninstr ;
2012-12-13 00:13:29 +01:00
GCStats stats ;
uint32 i ;
2013-01-29 21:52:43 +01:00
// Eface eface;
2012-12-13 00:13:29 +01:00
m = runtime_m ( ) ;
2014-07-19 10:53:52 +02:00
if ( runtime_debug . allocfreetrace )
runtime_tracegc ( ) ;
m - > traceback = 2 ;
2013-11-06 20:49:01 +01:00
t0 = args - > start_time ;
2014-06-07 00:37:27 +02:00
work . tstart = args - > start_time ;
2010-12-03 05:34:57 +01:00
2013-07-16 08:54:42 +02:00
if ( CollectStats )
runtime_memclr ( ( byte * ) & gcstats , sizeof ( gcstats ) ) ;
2012-12-22 02:15:33 +01:00
m - > locks + + ; // disable gc during mallocs in parforalloc
if ( work . markfor = = nil )
work . markfor = runtime_parforalloc ( MaxGcproc ) ;
m - > locks - - ;
2014-07-19 10:53:52 +02:00
t1 = 0 ;
if ( runtime_debug . gctrace )
t1 = runtime_nanotime ( ) ;
2014-06-07 00:37:27 +02:00
// Sweep what is not sweeped by bgsweep.
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 )
gcstats . npausesweep + + ;
2011-10-27 01:57:58 +02:00
work . nwait = 0 ;
work . ndone = 0 ;
2012-10-23 06:31:11 +02:00
work . nproc = runtime_gcprocs ( ) ;
2015-10-31 01:59:47 +01:00
runtime_parforsetup ( work . markfor , work . nproc , RootCount + runtime_allglen , false , & markroot_funcval ) ;
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 ) {
runtime_noteclear ( & work . alldone ) ;
runtime_helpgc ( work . nproc ) ;
}
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
t2 = 0 ;
if ( runtime_debug . gctrace )
t2 = runtime_nanotime ( ) ;
2013-01-29 21:52:43 +01:00
2013-07-16 08:54:42 +02:00
gchelperstart ( ) ;
2012-10-23 06:31:11 +02:00
runtime_parfordo ( work . markfor ) ;
2014-06-07 00:37:27 +02:00
scanblock ( nil , true ) ;
2011-10-27 01:57:58 +02:00
2014-07-19 10:53:52 +02:00
t3 = 0 ;
if ( runtime_debug . gctrace )
t3 = runtime_nanotime ( ) ;
2011-10-27 01:57:58 +02:00
2014-06-07 00:37:27 +02:00
bufferList [ m - > helpgc ] . busy = 0 ;
2012-10-23 06:31:11 +02:00
if ( work . nproc > 1 )
runtime_notesleep ( & work . alldone ) ;
2013-11-06 20:49:01 +01:00
cachestats ( ) ;
2014-06-07 00:37:27 +02:00
// next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
// estimate what was live heap size after previous GC (for tracing only)
heap0 = mstats . next_gc * 100 / ( gcpercent + 100 ) ;
// conservatively set next_gc to high value assuming that everything is live
// concurrent/lazy sweep will reduce this number while discovering new garbage
2015-09-15 14:32:31 +02:00
mstats . next_gc = mstats . heap_alloc + ( mstats . heap_alloc - runtime_stacks_sys ) * gcpercent / 100 ;
2011-03-17 00:05:44 +01:00
2013-01-29 21:52:43 +01:00
t4 = runtime_nanotime ( ) ;
2014-07-19 10:53:52 +02:00
mstats . last_gc = runtime_unixnanotime ( ) ; // must be Unix time to make sense to user
2013-01-29 21:52:43 +01:00
mstats . pause_ns [ mstats . numgc % nelem ( mstats . pause_ns ) ] = t4 - t0 ;
2015-10-31 01:59:47 +01:00
mstats . pause_end [ mstats . numgc % nelem ( mstats . pause_end ) ] = mstats . last_gc ;
2013-01-29 21:52:43 +01:00
mstats . pause_total_ns + = t4 - t0 ;
2010-12-03 05:34:57 +01:00
mstats . numgc + + ;
if ( mstats . debuggc )
2013-01-29 21:52:43 +01:00
runtime_printf ( " pause %D \n " , t4 - t0 ) ;
2011-10-27 01:57:58 +02:00
2013-11-06 20:49:01 +01:00
if ( runtime_debug . gctrace ) {
heap1 = mstats . heap_alloc ;
2014-07-19 10:53:52 +02:00
runtime_updatememstats ( & stats ) ;
if ( heap1 ! = mstats . heap_alloc ) {
runtime_printf ( " runtime: mstats skew: heap=%D/%D \n " , heap1 , mstats . heap_alloc ) ;
runtime_throw ( " mstats skew " ) ;
}
2014-06-07 00:37:27 +02:00
obj = mstats . nmalloc - mstats . nfree ;
2013-11-06 20:49:01 +01:00
2014-06-07 00:37:27 +02:00
stats . nprocyield + = work . markfor - > nprocyield ;
stats . nosyield + = work . markfor - > nosyield ;
stats . nsleep + = work . markfor - > nsleep ;
2013-11-06 20:49:01 +01:00
2014-07-19 10:53:52 +02:00
runtime_printf ( " gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects, "
2014-06-07 00:37:27 +02:00
" %d/%d/%d sweeps, "
2012-10-23 06:31:11 +02:00
" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields \n " ,
2014-07-19 10:53:52 +02:00
mstats . numgc , work . nproc , ( t1 - t0 ) / 1000 , ( t2 - t1 ) / 1000 , ( t3 - t2 ) / 1000 , ( t4 - t3 ) / 1000 ,
2014-06-07 00:37:27 +02:00
heap0 > > 20 , heap1 > > 20 , obj ,
2012-10-23 06:31:11 +02:00
mstats . nmalloc , mstats . nfree ,
2014-06-07 00:37:27 +02:00
sweep . nspan , gcstats . nbgsweep , gcstats . npausesweep ,
2012-10-23 06:31:11 +02:00
stats . nhandoff , stats . nhandoffcnt ,
2014-06-07 00:37:27 +02:00
work . markfor - > nsteal , work . markfor - > nstealcnt ,
2012-10-23 06:31:11 +02:00
stats . nprocyield , stats . nosyield , stats . nsleep ) ;
2014-06-07 00:37:27 +02:00
gcstats . nbgsweep = gcstats . npausesweep = 0 ;
2013-07-16 08:54:42 +02:00
if ( CollectStats ) {
runtime_printf ( " scan: %D bytes, %D objects, %D untyped, %D types from MSpan \n " ,
gcstats . nbytes , gcstats . obj . cnt , gcstats . obj . notype , gcstats . obj . typelookup ) ;
if ( gcstats . ptr . cnt ! = 0 )
runtime_printf ( " avg ptrbufsize: %D (%D/%D) \n " ,
gcstats . ptr . sum / gcstats . ptr . cnt , gcstats . ptr . sum , gcstats . ptr . cnt ) ;
if ( gcstats . obj . cnt ! = 0 )
runtime_printf ( " avg nobj: %D (%D/%D) \n " ,
gcstats . obj . sum / gcstats . obj . cnt , gcstats . obj . sum , gcstats . obj . cnt ) ;
runtime_printf ( " rescans: %D, %D bytes \n " , gcstats . rescan , gcstats . rescanbytes ) ;
runtime_printf ( " instruction counts: \n " ) ;
ninstr = 0 ;
for ( i = 0 ; i < nelem ( gcstats . instr ) ; i + + ) {
runtime_printf ( " \t %d: \t %D \n " , i , gcstats . instr [ i ] ) ;
ninstr + = gcstats . instr [ i ] ;
}
runtime_printf ( " \t total: \t %D \n " , ninstr ) ;
runtime_printf ( " putempty: %D, getfull: %D \n " , gcstats . putempty , gcstats . getfull ) ;
2013-11-06 20:49:01 +01:00
runtime_printf ( " markonly base lookup: bit %D word %D span %D \n " , gcstats . markonly . foundbit , gcstats . markonly . foundword , gcstats . markonly . foundspan ) ;
runtime_printf ( " flushptrbuf base lookup: bit %D word %D span %D \n " , gcstats . flushptrbuf . foundbit , gcstats . flushptrbuf . foundword , gcstats . flushptrbuf . foundspan ) ;
2013-07-16 08:54:42 +02:00
}
2011-03-17 00:05:44 +01:00
}
2012-10-23 06:31:11 +02:00
2014-06-07 00:37:27 +02:00
// We cache current runtime_mheap.allspans array in sweep.spans,
// because the former can be resized and freed.
// Otherwise we would need to take heap lock every time
// we want to convert span index to span pointer.
// Free the old cached array if necessary.
if ( sweep . spans & & sweep . spans ! = runtime_mheap . allspans )
runtime_SysFree ( sweep . spans , sweep . nspan * sizeof ( sweep . spans [ 0 ] ) , & mstats . other_sys ) ;
// Cache the current array.
runtime_mheap . sweepspans = runtime_mheap . allspans ;
runtime_mheap . sweepgen + = 2 ;
runtime_mheap . sweepdone = false ;
sweep . spans = runtime_mheap . allspans ;
sweep . nspan = runtime_mheap . nspan ;
sweep . spanidx = 0 ;
// Temporary disable concurrent sweep, because we see failures on builders.
2014-07-19 10:53:52 +02:00
if ( ConcurrentSweep & & ! args - > eagersweep ) {
2014-06-07 00:37:27 +02:00
runtime_lock ( & gclock ) ;
if ( sweep . g = = nil )
sweep . g = __go_go ( bgsweep , nil ) ;
else if ( sweep . parked ) {
sweep . parked = false ;
runtime_ready ( sweep . g ) ;
}
runtime_unlock ( & gclock ) ;
} else {
// Sweep all spans eagerly.
while ( runtime_sweepone ( ) ! = ( uintptr ) - 1 )
gcstats . npausesweep + + ;
2015-01-15 01:27:56 +01:00
// Do an additional mProf_GC, because all 'free' events are now real as well.
runtime_MProf_GC ( ) ;
2014-06-07 00:37:27 +02:00
}
2012-03-02 21:01:37 +01:00
runtime_MProf_GC ( ) ;
2014-07-19 10:53:52 +02:00
m - > traceback = 0 ;
2010-12-03 05:34:57 +01:00
}
2014-06-07 00:37:27 +02:00
extern uintptr runtime_sizeof_C_MStats
__asm__ ( GOSYM_PREFIX " runtime.Sizeof_C_MStats " ) ;
2012-02-09 09:19:58 +01:00
void runtime_ReadMemStats ( MStats * )
2013-01-24 20:44:23 +01:00
__asm__ ( GOSYM_PREFIX " runtime.ReadMemStats " ) ;
2011-09-16 17:47:21 +02:00
void
2012-02-09 09:19:58 +01:00
runtime_ReadMemStats ( MStats * stats )
2011-09-16 17:47:21 +02:00
{
2011-11-28 06:45:49 +01:00
M * m ;
2012-03-02 21:01:37 +01:00
// Have to acquire worldsema to stop the world,
2011-09-16 17:47:21 +02:00
// because stoptheworld can only be used by
// one goroutine at a time, and there might be
// a pending garbage collection already calling it.
2013-11-06 20:49:01 +01:00
runtime_semacquire ( & runtime_worldsema , false ) ;
2011-11-28 06:45:49 +01:00
m = runtime_m ( ) ;
2011-09-16 17:47:21 +02:00
m - > gcing = 1 ;
runtime_stoptheworld ( ) ;
2014-07-19 10:53:52 +02:00
runtime_updatememstats ( nil ) ;
2014-06-07 00:37:27 +02:00
// Size of the trailing by_size array differs between Go and C,
2016-08-30 23:07:47 +02:00
// _NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
2014-06-07 00:37:27 +02:00
runtime_memmove ( stats , & mstats , runtime_sizeof_C_MStats ) ;
2011-09-16 17:47:21 +02:00
m - > gcing = 0 ;
2013-11-06 20:49:01 +01:00
m - > locks + + ;
2012-03-02 21:01:37 +01:00
runtime_semrelease ( & runtime_worldsema ) ;
2012-10-23 06:31:11 +02:00
runtime_starttheworld ( ) ;
2013-11-06 20:49:01 +01:00
m - > locks - - ;
2011-09-16 17:47:21 +02:00
}
2013-07-16 08:54:42 +02:00
void runtime_debug_readGCStats ( Slice * )
__asm__ ( " runtime_debug.readGCStats " ) ;
void
runtime_debug_readGCStats ( Slice * pauses )
{
uint64 * p ;
uint32 i , n ;
// Calling code in runtime/debug should make the slice large enough.
if ( ( size_t ) pauses - > cap < nelem ( mstats . pause_ns ) + 3 )
runtime_throw ( " runtime: short slice passed to readGCStats " ) ;
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = ( uint64 * ) pauses - > array ;
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
n = mstats . numgc ;
if ( n > nelem ( mstats . pause_ns ) )
n = nelem ( mstats . pause_ns ) ;
// The pause buffer is circular. The most recent pause is at
// pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
// from there to go back farther in time. We deliver the times
// most recent first (in p[0]).
for ( i = 0 ; i < n ; i + + )
p [ i ] = mstats . pause_ns [ ( mstats . numgc - 1 - i ) % nelem ( mstats . pause_ns ) ] ;
p [ n ] = mstats . last_gc ;
p [ n + 1 ] = mstats . numgc ;
p [ n + 2 ] = mstats . pause_total_ns ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
pauses - > __count = n + 3 ;
}
2014-07-12 02:01:09 +02:00
int32
runtime_setgcpercent ( int32 in ) {
int32 out ;
2013-07-16 08:54:42 +02:00
2013-11-06 20:49:01 +01:00
runtime_lock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
if ( gcpercent = = GcpercentUnknown )
gcpercent = readgogc ( ) ;
out = gcpercent ;
if ( in < 0 )
in = - 1 ;
gcpercent = in ;
2013-11-06 20:49:01 +01:00
runtime_unlock ( & runtime_mheap ) ;
2013-07-16 08:54:42 +02:00
return out ;
}
static void
gchelperstart ( void )
{
M * m ;
m = runtime_m ( ) ;
if ( m - > helpgc < 0 | | m - > helpgc > = MaxGcproc )
runtime_throw ( " gchelperstart: bad m->helpgc " ) ;
if ( runtime_xchg ( & bufferList [ m - > helpgc ] . busy , 1 ) )
runtime_throw ( " gchelperstart: already busy " ) ;
2013-11-06 20:49:01 +01:00
if ( runtime_g ( ) ! = m - > g0 )
runtime_throw ( " gchelper not running on g0 stack " ) ;
2013-07-16 08:54:42 +02:00
}
2010-12-03 05:34:57 +01:00
static void
2011-11-28 06:45:49 +01:00
runfinq ( void * dummy __attribute__ ( ( unused ) ) )
2010-12-03 05:34:57 +01:00
{
2011-10-27 01:57:58 +02:00
Finalizer * f ;
FinBlock * fb , * next ;
uint32 i ;
2013-11-06 20:49:01 +01:00
Eface ef ;
Iface iface ;
2010-12-03 05:34:57 +01:00
2014-07-19 10:53:52 +02:00
// This function blocks for long periods of time, and because it is written in C
// we have no liveness information. Zero everything so that uninitialized pointers
// do not cause memory leaks.
f = nil ;
fb = nil ;
next = nil ;
i = 0 ;
ef . __type_descriptor = nil ;
ef . __object = nil ;
// force flush to memory
USED ( & f ) ;
USED ( & fb ) ;
USED ( & next ) ;
USED ( & i ) ;
USED ( & ef ) ;
2010-12-03 05:34:57 +01:00
for ( ; ; ) {
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
fb = finq ;
2010-12-03 05:34:57 +01:00
finq = nil ;
2011-10-27 01:57:58 +02:00
if ( fb = = nil ) {
2014-07-19 10:53:52 +02:00
runtime_fingwait = true ;
runtime_g ( ) - > isbackground = true ;
runtime_parkunlock ( & finlock , " finalizer wait " ) ;
runtime_g ( ) - > isbackground = false ;
2010-12-03 05:34:57 +01:00
continue ;
}
2014-07-19 10:53:52 +02:00
runtime_unlock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
for ( ; fb ; fb = next ) {
next = fb - > next ;
for ( i = 0 ; i < ( uint32 ) fb - > cnt ; i + + ) {
2013-11-06 20:49:01 +01:00
const Type * fint ;
2013-09-03 23:52:37 +02:00
void * param ;
2011-10-27 01:57:58 +02:00
f = & fb - > fin [ i ] ;
2013-11-06 20:49:01 +01:00
fint = ( ( const Type * * ) f - > ft - > __in . array ) [ 0 ] ;
compiler, runtime: replace hashmap code with Go 1.7 hashmap
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
2016-09-21 22:58:51 +02:00
if ( ( fint - > __code & kindMask ) = = kindPtr ) {
2013-11-06 20:49:01 +01:00
// direct use of pointer
param = & f - > arg ;
} else if ( ( ( const InterfaceType * ) fint ) - > __methods . __count = = 0 ) {
// convert to empty interface
2014-06-07 00:37:27 +02:00
ef . __type_descriptor = ( const Type * ) f - > ot ;
2013-11-06 20:49:01 +01:00
ef . __object = f - > arg ;
param = & ef ;
} else {
// convert to interface with methods
iface . __methods = __go_convert_interface_2 ( ( const Type * ) fint ,
( const Type * ) f - > ot ,
1 ) ;
iface . __object = f - > arg ;
if ( iface . __methods = = nil )
runtime_throw ( " invalid type conversion in runfinq " ) ;
param = & iface ;
}
2013-09-03 23:52:37 +02:00
reflect_call ( f - > ft , f - > fn , 0 , 0 , & param , nil ) ;
2011-10-27 01:57:58 +02:00
f - > fn = nil ;
f - > arg = nil ;
2013-11-06 20:49:01 +01:00
f - > ot = nil ;
2011-10-27 01:57:58 +02:00
}
fb - > cnt = 0 ;
2014-07-19 10:53:52 +02:00
runtime_lock ( & finlock ) ;
2011-10-27 01:57:58 +02:00
fb - > next = finc ;
finc = fb ;
2014-07-19 10:53:52 +02:00
runtime_unlock ( & finlock ) ;
2010-12-03 05:34:57 +01:00
}
2014-07-19 10:53:52 +02:00
// Zero everything that's dead, to avoid memory leaks.
// See comment at top of function.
f = nil ;
fb = nil ;
next = nil ;
i = 0 ;
ef . __type_descriptor = nil ;
ef . __object = nil ;
2010-12-03 05:34:57 +01:00
runtime_gc ( 1 ) ; // trigger another gc to clean up the finalized objects, if possible
}
}
2014-07-19 10:53:52 +02:00
void
runtime_createfing ( void )
{
if ( fing ! = nil )
return ;
// Here we use gclock instead of finlock,
// because newproc1 can allocate, which can cause on-demand span sweep,
// which can queue finalizers, which would deadlock.
runtime_lock ( & gclock ) ;
if ( fing = = nil )
fing = __go_go ( runfinq , nil ) ;
runtime_unlock ( & gclock ) ;
}
G *
runtime_wakefing ( void )
{
G * res ;
res = nil ;
runtime_lock ( & finlock ) ;
if ( runtime_fingwait & & runtime_fingwake ) {
runtime_fingwait = false ;
runtime_fingwake = false ;
res = fing ;
}
runtime_unlock ( & finlock ) ;
return res ;
}
2011-03-17 00:05:44 +01:00
void
2014-06-05 01:15:33 +02:00
runtime_marknogc ( void * v )
2011-03-17 00:05:44 +01:00
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2014-06-05 01:15:33 +02:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b = ( * b & ~ ( bitAllocated < < shift ) ) | bitBlockBoundary < < shift ;
2014-06-05 01:15:33 +02:00
}
void
runtime_markscan ( void * v )
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b | = bitScan < < shift ;
2011-03-17 00:05:44 +01:00
}
2014-07-19 10:53:52 +02:00
// mark the block at v as freed.
2011-03-17 00:05:44 +01:00
void
2014-07-19 10:53:52 +02:00
runtime_markfreed ( void * v )
2011-03-17 00:05:44 +01:00
{
2014-07-19 10:53:52 +02:00
uintptr * b , off , shift ;
2011-03-17 00:05:44 +01:00
2012-05-24 22:44:34 +02:00
if ( 0 )
2014-07-19 10:53:52 +02:00
runtime_printf ( " markfreed %p \n " , v ) ;
2011-03-17 00:05:44 +01:00
2014-07-19 10:53:52 +02:00
if ( ( byte * ) v > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2013-11-06 20:49:01 +01:00
runtime_throw ( " markfreed: bad pointer " ) ;
2011-03-17 00:05:44 +01:00
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
* b = ( * b & ~ ( bitMask < < shift ) ) | ( bitAllocated < < shift ) ;
2011-03-17 00:05:44 +01:00
}
// check that the block at v of size n is marked freed.
void
runtime_checkfreed ( void * v , uintptr n )
{
uintptr * b , bits , off , shift ;
if ( ! runtime_checking )
return ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
return ; // not allocated, so okay
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) v - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
bits = * b > > shift ;
if ( ( bits & bitAllocated ) ! = 0 ) {
runtime_printf ( " checkfreed %p+%p: off=%p have=%p \n " ,
2012-05-24 22:44:34 +02:00
v , n , off , bits & bitMask ) ;
2011-03-17 00:05:44 +01:00
runtime_throw ( " checkfreed: not freed " ) ;
}
}
// mark the span of memory at v as having n blocks of the given size.
// if leftover is true, there is left over space at the end of the span.
void
runtime_markspan ( void * v , uintptr size , uintptr n , bool leftover )
{
2014-07-19 10:53:52 +02:00
uintptr * b , * b0 , off , shift , i , x ;
2011-03-17 00:05:44 +01:00
byte * p ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + size * n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
2014-06-05 01:15:33 +02:00
if ( runtime_checking ) {
// bits should be all zero at the start
off = ( byte * ) v + size - runtime_mheap . arena_start ;
b = ( uintptr * ) ( runtime_mheap . arena_start - off / wordsPerBitmapWord ) ;
for ( i = 0 ; i < size / PtrSize / wordsPerBitmapWord ; i + + ) {
if ( b [ i ] ! = 0 )
runtime_throw ( " markspan: span bits not zero " ) ;
}
}
2011-03-17 00:05:44 +01:00
p = v ;
if ( leftover ) // mark a boundary just past end of last block too
n + + ;
2014-07-19 10:53:52 +02:00
b0 = nil ;
x = 0 ;
2011-03-17 00:05:44 +01:00
for ( ; n - - > 0 ; p + = size ) {
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
2013-11-06 20:49:01 +01:00
off = ( uintptr * ) p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
shift = off % wordsPerBitmapWord ;
2014-07-19 10:53:52 +02:00
if ( b0 ! = b ) {
if ( b0 ! = nil )
* b0 = x ;
b0 = b ;
x = 0 ;
}
x | = bitAllocated < < shift ;
2011-03-17 00:05:44 +01:00
}
2014-07-19 10:53:52 +02:00
* b0 = x ;
2011-03-17 00:05:44 +01:00
}
// unmark the span of memory at v of length n bytes.
void
runtime_unmarkspan ( void * v , uintptr n )
{
uintptr * p , * b , off ;
2013-11-06 20:49:01 +01:00
if ( ( byte * ) v + n > ( byte * ) runtime_mheap . arena_used | | ( byte * ) v < runtime_mheap . arena_start )
2011-03-17 00:05:44 +01:00
runtime_throw ( " markspan: bad pointer " ) ;
p = v ;
2013-11-06 20:49:01 +01:00
off = p - ( uintptr * ) runtime_mheap . arena_start ; // word offset
2011-03-17 00:05:44 +01:00
if ( off % wordsPerBitmapWord ! = 0 )
runtime_throw ( " markspan: unaligned pointer " ) ;
2013-11-06 20:49:01 +01:00
b = ( uintptr * ) runtime_mheap . arena_start - off / wordsPerBitmapWord - 1 ;
2011-03-17 00:05:44 +01:00
n / = PtrSize ;
if ( n % wordsPerBitmapWord ! = 0 )
runtime_throw ( " unmarkspan: unaligned length " ) ;
// Okay to use non-atomic ops here, because we control
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
n / = wordsPerBitmapWord ;
while ( n - - > 0 )
* b - - = 0 ;
}
void
runtime_MHeap_MapBits ( MHeap * h )
{
2012-04-20 06:58:26 +02:00
size_t page_size ;
2011-03-17 00:05:44 +01:00
// Caller has added extra mappings to the arena.
// Add extra mappings of bitmap words as needed.
// We allocate extra bitmap pieces in chunks of bitmapChunk.
enum {
bitmapChunk = 8192
} ;
uintptr n ;
2011-10-27 01:57:58 +02:00
2011-03-17 00:05:44 +01:00
n = ( h - > arena_used - h - > arena_start ) / wordsPerBitmapWord ;
2013-11-06 20:49:01 +01:00
n = ROUND ( n , bitmapChunk ) ;
2014-07-19 10:53:52 +02:00
n = ROUND ( n , PageSize ) ;
page_size = getpagesize ( ) ;
n = ROUND ( n , page_size ) ;
2011-03-17 00:05:44 +01:00
if ( h - > bitmap_mapped > = n )
return ;
2014-07-19 10:53:52 +02:00
runtime_SysMap ( h - > arena_start - n , n - h - > bitmap_mapped , h - > arena_reserved , & mstats . gc_sys ) ;
2011-03-17 00:05:44 +01:00
h - > bitmap_mapped = n ;
}
2015-10-31 01:59:47 +01:00
// typedmemmove copies a value of type t to dst from src.
extern void typedmemmove ( const Type * td , void * dst , const void * src )
__asm__ ( GOSYM_PREFIX " reflect.typedmemmove " ) ;
void
typedmemmove ( const Type * td , void * dst , const void * src )
{
runtime_memmove ( dst , src , td - > __size ) ;
}
// typedslicecopy copies a slice of elemType values from src to dst,
// returning the number of elements copied.
extern intgo typedslicecopy ( const Type * elem , Slice dst , Slice src )
__asm__ ( GOSYM_PREFIX " reflect.typedslicecopy " ) ;
intgo
typedslicecopy ( const Type * elem , Slice dst , Slice src )
{
intgo n ;
void * dstp ;
void * srcp ;
n = dst . __count ;
if ( n > src . __count )
n = src . __count ;
if ( n = = 0 )
return 0 ;
dstp = dst . __values ;
srcp = src . __values ;
memmove ( dstp , srcp , ( uintptr_t ) n * elem - > __size ) ;
return n ;
}