202 lines
6.3 KiB
Go
202 lines
6.3 KiB
Go
// Copyright 2016 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// The gccgo version of mem_*.go.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"unsafe"
|
|
)
|
|
|
|
// Functions called by C code.
|
|
//go:linkname sysAlloc
|
|
//go:linkname sysFree
|
|
|
|
//extern mmap
|
|
func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) unsafe.Pointer
|
|
|
|
//extern munmap
|
|
func munmap(addr unsafe.Pointer, length uintptr) int32
|
|
|
|
//extern madvise
|
|
func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32
|
|
|
|
var mmapFD = int32(-1)
|
|
|
|
var devZero = []byte("/dev/zero\x00")
|
|
|
|
func init() {
|
|
if _MAP_ANON == 0 {
|
|
mmapFD = open(&devZero[0], 0 /* O_RDONLY */, 0)
|
|
if mmapFD < 0 {
|
|
println("open /dev/zero: errno=", errno())
|
|
exit(2)
|
|
}
|
|
}
|
|
}
|
|
|
|
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) (unsafe.Pointer, int) {
|
|
p := sysMmap(addr, n, prot, flags, fd, off)
|
|
if uintptr(p) == _MAP_FAILED {
|
|
return nil, errno()
|
|
}
|
|
return p, 0
|
|
}
|
|
|
|
// Don't split the stack as this method may be invoked without a valid G, which
|
|
// prevents us from allocating more stack.
|
|
//go:nosplit
|
|
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
|
|
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0)
|
|
if err != 0 {
|
|
if err == _EACCES {
|
|
print("runtime: mmap: access denied\n")
|
|
exit(2)
|
|
}
|
|
if err == _EAGAIN {
|
|
print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
|
|
exit(2)
|
|
}
|
|
return nil
|
|
}
|
|
mSysStatInc(sysStat, n)
|
|
return p
|
|
}
|
|
|
|
func sysUnused(v unsafe.Pointer, n uintptr) {
|
|
// By default, Linux's "transparent huge page" support will
|
|
// merge pages into a huge page if there's even a single
|
|
// present regular page, undoing the effects of the DONTNEED
|
|
// below. On amd64, that means khugepaged can turn a single
|
|
// 4KB page to 2MB, bloating the process's RSS by as much as
|
|
// 512X. (See issue #8832 and Linux kernel bug
|
|
// https://bugzilla.kernel.org/show_bug.cgi?id=93111)
|
|
//
|
|
// To work around this, we explicitly disable transparent huge
|
|
// pages when we release pages of the heap. However, we have
|
|
// to do this carefully because changing this flag tends to
|
|
// split the VMA (memory mapping) containing v in to three
|
|
// VMAs in order to track the different values of the
|
|
// MADV_NOHUGEPAGE flag in the different regions. There's a
|
|
// default limit of 65530 VMAs per address space (sysctl
|
|
// vm.max_map_count), so we must be careful not to create too
|
|
// many VMAs (see issue #12233).
|
|
//
|
|
// Since huge pages are huge, there's little use in adjusting
|
|
// the MADV_NOHUGEPAGE flag on a fine granularity, so we avoid
|
|
// exploding the number of VMAs by only adjusting the
|
|
// MADV_NOHUGEPAGE flag on a large granularity. This still
|
|
// gets most of the benefit of huge pages while keeping the
|
|
// number of VMAs under control. With hugePageSize = 2MB, even
|
|
// a pessimal heap can reach 128GB before running out of VMAs.
|
|
if physHugePageSize != 0 && _MADV_NOHUGEPAGE != 0 {
|
|
// If it's a large allocation, we want to leave huge
|
|
// pages enabled. Hence, we only adjust the huge page
|
|
// flag on the huge pages containing v and v+n-1, and
|
|
// only if those aren't aligned.
|
|
var head, tail uintptr
|
|
if uintptr(v)%physHugePageSize != 0 {
|
|
// Compute huge page containing v.
|
|
head = uintptr(v) &^ (physHugePageSize - 1)
|
|
}
|
|
if (uintptr(v)+n)%physHugePageSize != 0 {
|
|
// Compute huge page containing v+n-1.
|
|
tail = (uintptr(v) + n - 1) &^ (physHugePageSize - 1)
|
|
}
|
|
|
|
// Note that madvise will return EINVAL if the flag is
|
|
// already set, which is quite likely. We ignore
|
|
// errors.
|
|
if head != 0 && head+physHugePageSize == tail {
|
|
// head and tail are different but adjacent,
|
|
// so do this in one call.
|
|
madvise(unsafe.Pointer(head), 2*physHugePageSize, _MADV_NOHUGEPAGE)
|
|
} else {
|
|
// Advise the huge pages containing v and v+n-1.
|
|
if head != 0 {
|
|
madvise(unsafe.Pointer(head), physHugePageSize, _MADV_NOHUGEPAGE)
|
|
}
|
|
if tail != 0 && tail != head {
|
|
madvise(unsafe.Pointer(tail), physHugePageSize, _MADV_NOHUGEPAGE)
|
|
}
|
|
}
|
|
}
|
|
|
|
if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 {
|
|
// madvise will round this to any physical page
|
|
// *covered* by this range, so an unaligned madvise
|
|
// will release more memory than intended.
|
|
throw("unaligned sysUnused")
|
|
}
|
|
|
|
if _MADV_DONTNEED != 0 {
|
|
madvise(v, n, _MADV_DONTNEED)
|
|
} else if _MADV_FREE != 0 {
|
|
madvise(v, n, _MADV_FREE)
|
|
}
|
|
}
|
|
|
|
func sysUsed(v unsafe.Pointer, n uintptr) {
|
|
// Partially undo the NOHUGEPAGE marks from sysUnused
|
|
// for whole huge pages between v and v+n. This may
|
|
// leave huge pages off at the end points v and v+n
|
|
// even though allocations may cover these entire huge
|
|
// pages. We could detect this and undo NOHUGEPAGE on
|
|
// the end points as well, but it's probably not worth
|
|
// the cost because when neighboring allocations are
|
|
// freed sysUnused will just set NOHUGEPAGE again.
|
|
sysHugePage(v, n)
|
|
}
|
|
|
|
func sysHugePage(v unsafe.Pointer, n uintptr) {
|
|
if physHugePageSize != 0 && _MADV_HUGEPAGE != 0 {
|
|
// Round v up to a huge page boundary.
|
|
beg := (uintptr(v) + (physHugePageSize - 1)) &^ (physHugePageSize - 1)
|
|
// Round v+n down to a huge page boundary.
|
|
end := (uintptr(v) + n) &^ (physHugePageSize - 1)
|
|
|
|
if beg < end {
|
|
madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Don't split the stack as this function may be invoked without a valid G,
|
|
// which prevents us from allocating more stack.
|
|
//go:nosplit
|
|
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
|
|
mSysStatDec(sysStat, n)
|
|
munmap(v, n)
|
|
}
|
|
|
|
func sysFault(v unsafe.Pointer, n uintptr) {
|
|
mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, mmapFD, 0)
|
|
}
|
|
|
|
func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer {
|
|
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0)
|
|
if err != 0 {
|
|
return nil
|
|
}
|
|
return p
|
|
}
|
|
|
|
func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) {
|
|
mSysStatInc(sysStat, n)
|
|
|
|
if GOOS == "aix" {
|
|
// AIX does not allow mapping a range that is already mapped.
|
|
// So always unmap first even if it is already unmapped.
|
|
munmap(v, n)
|
|
}
|
|
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, mmapFD, 0)
|
|
if err == _ENOMEM {
|
|
throw("runtime: out of memory")
|
|
}
|
|
if p != v || err != 0 {
|
|
throw("runtime: cannot map pages in arena address space")
|
|
}
|
|
}
|