4a2bb7fcb0
This change removes the gccgo-specific hashmap code and replaces it with
the hashmap code from the Go 1.7 runtime. The Go 1.7 hashmap code is
more efficient, does a better job on details like when to update a key,
and provides some support against denial-of-service attacks.
The compiler is changed to call the new hashmap functions instead of the
old ones.
The compiler now tracks which types are reflexive and which require
updating when used as a map key, and records the information in map type
descriptors.
Map_index_expression is simplified. The special case for a map index on
the right hand side of a tuple expression has been unnecessary for some
time, and is removed. The support for specially marking a map index as
an lvalue is removed, in favor of lowering an assignment to a map index
into a function call. The long-obsolete support for a map index of a
pointer to a map is removed.
The __go_new_map_big function (known to the compiler as
Runtime::MAKEMAPBIG) is no longer needed, as the new runtime.makemap
function takes an int64 hint argument.
The old map descriptor type and supporting expression is removed.
The compiler was still supporting the long-obsolete syntax `m[k] = 0,
false` to delete a value from a map. That is now removed, requiring a
change to one of the gccgo-specific tests.
The builtin len function applied to a map or channel p is now compiled
as `p == nil ? 0 : *(*int)(p)`. The __go_chan_len function (known to
the compiler as Runtime::CHAN_LEN) is removed.
Support for a shared zero value for maps to large value types is
introduced, along the lines of the gc compiler. The zero value is
handled as a common variable.
The hash function is changed to take a seed argument, changing the
runtime hash functions and the compiler-generated hash functions.
Unlike the gc compiler, both the hash and equal functions continue to
take the type length.
Types that can not be compared now store nil for the hash and equal
functions, rather than pointing to functions that throw. Interface hash
and comparison functions now check explicitly for nil. This matches the
gc compiler and permits a simple implementation for ismapkey.
The compiler is changed to permit marking struct and array types as
incomparable, meaning that they have no hash or equal function. We use
this for thunk types, removing the existing special code to avoid
generating hash/equal functions for them.
The C runtime code adds memclr, memequal, and memmove functions.
The hashmap code uses go:linkname comments to make the functions
visible, as otherwise the compiler would discard them.
The hashmap code comments out the unused reference to the address of the
first parameter in the race code, as otherwise the compiler thinks that
the parameter escapes and copies it onto the heap. This is probably not
needed when we enable escape analysis.
Several runtime map tests that ere previously skipped for gccgo are now
run.
The Go runtime picks up type kind information and stubs. The type kind
information causes the generated runtime header file to define some
constants, including `empty`, and the C code is adjusted accordingly.
A Go-callable version of runtime.throw, that takes a Go string, is
added to be called from the hashmap code.
Reviewed-on: https://go-review.googlesource.com/29447
* go.go-torture/execute/map-1.go: Replace old map deletion syntax
with call to builtin delete function.
From-SVN: r240334
399 lines
12 KiB
Go
399 lines
12 KiB
Go
// Copyright 2014 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"runtime/internal/sys"
|
|
"unsafe"
|
|
)
|
|
|
|
func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_fast32))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
var b *bmap
|
|
if h.B == 0 {
|
|
// One-bucket table. No need to hash.
|
|
b = (*bmap)(h.buckets)
|
|
} else {
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
|
|
if k != key {
|
|
continue
|
|
}
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize))
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_fast32))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
var b *bmap
|
|
if h.B == 0 {
|
|
// One-bucket table. No need to hash.
|
|
b = (*bmap)(h.buckets)
|
|
} else {
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
k := *((*uint32)(add(unsafe.Pointer(b), dataOffset+i*4)))
|
|
if k != key {
|
|
continue
|
|
}
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.valuesize)), true
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_fast64))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
var b *bmap
|
|
if h.B == 0 {
|
|
// One-bucket table. No need to hash.
|
|
b = (*bmap)(h.buckets)
|
|
} else {
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
|
|
if k != key {
|
|
continue
|
|
}
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize))
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_fast64))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
var b *bmap
|
|
if h.B == 0 {
|
|
// One-bucket table. No need to hash.
|
|
b = (*bmap)(h.buckets)
|
|
} else {
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&key)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
k := *((*uint64)(add(unsafe.Pointer(b), dataOffset+i*8)))
|
|
if k != key {
|
|
continue
|
|
}
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.valuesize)), true
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_faststr))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
key := stringStructOf(&ky)
|
|
if h.B == 0 {
|
|
// One-bucket table.
|
|
b := (*bmap)(h.buckets)
|
|
if key.len < 32 {
|
|
// short key, doing lots of comparisons is ok
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
|
|
}
|
|
}
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
// long key, try not to do more comparisons than necessary
|
|
keymaybe := uintptr(bucketCnt)
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
|
|
}
|
|
// check first 4 bytes
|
|
// TODO: on amd64/386 at least, make this compile to one 4-byte comparison instead of
|
|
// four 1-byte comparisons.
|
|
if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) {
|
|
continue
|
|
}
|
|
// check last 4 bytes
|
|
if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) {
|
|
continue
|
|
}
|
|
if keymaybe != bucketCnt {
|
|
// Two keys are potential matches. Use hash to distinguish them.
|
|
goto dohash
|
|
}
|
|
keymaybe = i
|
|
}
|
|
if keymaybe != bucketCnt {
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*sys.PtrSize))
|
|
if memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+keymaybe*uintptr(t.valuesize))
|
|
}
|
|
}
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
dohash:
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
top := uint8(hash >> (sys.PtrSize*8 - 8))
|
|
if top < minTopHash {
|
|
top += minTopHash
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x != top {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize))
|
|
}
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
|
|
if raceenabled && h != nil {
|
|
callerpc := getcallerpc(unsafe.Pointer( /* &t */ nil))
|
|
racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_faststr))
|
|
}
|
|
if h == nil || h.count == 0 {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
if h.flags&hashWriting != 0 {
|
|
throw("concurrent map read and map write")
|
|
}
|
|
key := stringStructOf(&ky)
|
|
if h.B == 0 {
|
|
// One-bucket table.
|
|
b := (*bmap)(h.buckets)
|
|
if key.len < 32 {
|
|
// short key, doing lots of comparisons is ok
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)), true
|
|
}
|
|
}
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
// long key, try not to do more comparisons than necessary
|
|
keymaybe := uintptr(bucketCnt)
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x == empty {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)), true
|
|
}
|
|
// check first 4 bytes
|
|
if *((*[4]byte)(key.str)) != *((*[4]byte)(k.str)) {
|
|
continue
|
|
}
|
|
// check last 4 bytes
|
|
if *((*[4]byte)(add(key.str, uintptr(key.len)-4))) != *((*[4]byte)(add(k.str, uintptr(key.len)-4))) {
|
|
continue
|
|
}
|
|
if keymaybe != bucketCnt {
|
|
// Two keys are potential matches. Use hash to distinguish them.
|
|
goto dohash
|
|
}
|
|
keymaybe = i
|
|
}
|
|
if keymaybe != bucketCnt {
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+keymaybe*2*sys.PtrSize))
|
|
if memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+keymaybe*uintptr(t.valuesize)), true
|
|
}
|
|
}
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
dohash:
|
|
hash := t.key.hashfn(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0), uintptr(t.keysize))
|
|
m := uintptr(1)<<h.B - 1
|
|
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
|
|
if c := h.oldbuckets; c != nil {
|
|
oldb := (*bmap)(add(c, (hash&(m>>1))*uintptr(t.bucketsize)))
|
|
if !evacuated(oldb) {
|
|
b = oldb
|
|
}
|
|
}
|
|
top := uint8(hash >> (sys.PtrSize*8 - 8))
|
|
if top < minTopHash {
|
|
top += minTopHash
|
|
}
|
|
for {
|
|
for i := uintptr(0); i < bucketCnt; i++ {
|
|
x := *((*uint8)(add(unsafe.Pointer(b), i))) // b.topbits[i] without the bounds check
|
|
if x != top {
|
|
continue
|
|
}
|
|
k := (*stringStruct)(add(unsafe.Pointer(b), dataOffset+i*2*sys.PtrSize))
|
|
if k.len != key.len {
|
|
continue
|
|
}
|
|
if k.str == key.str || memequal(k.str, key.str, uintptr(key.len)) {
|
|
return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*sys.PtrSize+i*uintptr(t.valuesize)), true
|
|
}
|
|
}
|
|
b = b.overflow(t)
|
|
if b == nil {
|
|
return unsafe.Pointer(&zeroVal[0]), false
|
|
}
|
|
}
|
|
}
|