2013-07-15 10:13:36 +02:00
|
|
|
|
/*
|
2022-01-31 22:02:06 +01:00
|
|
|
|
* Copyright (c) 2013-2022 Joris Vink <joris@coders.se>
|
2013-07-15 10:13:36 +02:00
|
|
|
|
*
|
|
|
|
|
* Permission to use, copy, modify, and distribute this software for any
|
|
|
|
|
* purpose with or without fee is hereby granted, provided that the above
|
|
|
|
|
* copyright notice and this permission notice appear in all copies.
|
|
|
|
|
*
|
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
|
|
|
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
|
|
|
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
|
|
|
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
|
|
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
|
|
|
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
|
|
|
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
|
*/
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
/*
|
|
|
|
|
* A kore_pool is a memory pool containing fixed-sized objects that
|
|
|
|
|
* can quickly be obtained by a caller via kore_pool_get() and returned
|
|
|
|
|
* via kore_pool_put().
|
|
|
|
|
*
|
|
|
|
|
* Each entry in a pool will have a canary at the end that is used to
|
|
|
|
|
* catch any potential overruns when the entry is returned to the pool.
|
|
|
|
|
*
|
|
|
|
|
* If memory pool guards are enabled three additional things happen:
|
|
|
|
|
*
|
|
|
|
|
* 1) The metadata is placed at the start of a page instead
|
|
|
|
|
* of right before the returned user pointer.
|
|
|
|
|
*
|
|
|
|
|
* 2) Each pool entry gets a guard page at the end of its allocation
|
|
|
|
|
* that is marked as PROT_NONE. Touching a guard page will cause
|
|
|
|
|
* the application to receive a SIGSEGV.
|
|
|
|
|
*
|
|
|
|
|
* 3) Entries are only marked PROT_READ | PROT_WRITE when they are
|
|
|
|
|
* obtained with kore_pool_get(). Their memory protection is
|
|
|
|
|
* changed to PROT_NONE when returned to the pool via kore_pool_get().
|
|
|
|
|
*
|
|
|
|
|
* Caveats:
|
|
|
|
|
* Pools are designed to live for the entire lifetime of a Kore process
|
|
|
|
|
* until it will exit and are therefor not properly cleaned up when exit
|
|
|
|
|
* time arrives.
|
|
|
|
|
*/
|
|
|
|
|
|
2019-03-06 09:29:46 +01:00
|
|
|
|
#include <sys/types.h>
|
2016-07-12 13:54:14 +02:00
|
|
|
|
#include <sys/mman.h>
|
2013-07-15 10:13:36 +02:00
|
|
|
|
#include <sys/queue.h>
|
|
|
|
|
|
2016-07-12 13:54:14 +02:00
|
|
|
|
#include <stdint.h>
|
|
|
|
|
|
2013-07-15 10:13:36 +02:00
|
|
|
|
#include "kore.h"
|
|
|
|
|
|
2018-10-30 10:41:49 +01:00
|
|
|
|
#define POOL_MIN_ELEMENTS 16
|
|
|
|
|
|
2013-07-15 10:13:36 +02:00
|
|
|
|
#define POOL_ELEMENT_BUSY 0
|
|
|
|
|
#define POOL_ELEMENT_FREE 1
|
|
|
|
|
|
2016-07-12 16:29:30 +02:00
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
static void pool_lock(struct kore_pool *);
|
|
|
|
|
static void pool_unlock(struct kore_pool *);
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
static void pool_grow(struct kore_pool *, size_t);
|
|
|
|
|
|
|
|
|
|
static void pool_mark_entry_rw(struct kore_pool *, void *);
|
|
|
|
|
static void pool_mark_entry_none(struct kore_pool *, void *);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
|
|
|
|
void
|
2014-08-04 12:40:21 +02:00
|
|
|
|
kore_pool_init(struct kore_pool *pool, const char *name,
|
2016-07-12 14:01:02 +02:00
|
|
|
|
size_t len, size_t elm)
|
2013-07-15 10:13:36 +02:00
|
|
|
|
{
|
2023-01-05 22:47:29 +01:00
|
|
|
|
long pagesz;
|
|
|
|
|
|
2018-10-30 10:41:49 +01:00
|
|
|
|
if (elm < POOL_MIN_ELEMENTS)
|
|
|
|
|
elm = POOL_MIN_ELEMENTS;
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if ((pagesz = sysconf(_SC_PAGESIZE)) == -1)
|
|
|
|
|
fatal("%s: sysconf: %s", __func__, errno_s);
|
|
|
|
|
|
2016-07-12 13:54:14 +02:00
|
|
|
|
if ((pool->name = strdup(name)) == NULL)
|
|
|
|
|
fatal("kore_pool_init: strdup %s", errno_s);
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool->uselen = len;
|
|
|
|
|
|
|
|
|
|
len = len + sizeof(u_int64_t);
|
|
|
|
|
len = (len + (16 - 1)) & ~(16 - 1);
|
|
|
|
|
|
|
|
|
|
pool->elmlen = len;
|
2021-12-11 22:35:37 +01:00
|
|
|
|
|
2016-07-12 16:29:30 +02:00
|
|
|
|
pool->lock = 0;
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool->freelist = NULL;
|
|
|
|
|
pool->pagesz = pagesz;
|
2018-10-30 10:36:18 +01:00
|
|
|
|
pool->growth = elm * 0.25f;
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool->canary = (u_int64_t)kore_platform_random_uint32() << 32 |
|
|
|
|
|
kore_platform_random_uint32();
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (kore_mem_guard) {
|
|
|
|
|
pool->memsz = pool->pagesz * 2;
|
|
|
|
|
|
|
|
|
|
while (pool->elmlen >
|
|
|
|
|
pool->pagesz - sizeof(struct kore_pool_entry)) {
|
|
|
|
|
pool->memsz += pool->pagesz;
|
|
|
|
|
pool->elmlen -= MIN(pool->elmlen, pool->pagesz);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pool->elmlen = len;
|
|
|
|
|
} else {
|
|
|
|
|
pool->memsz = pool->elmlen;
|
|
|
|
|
}
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool_grow(pool, elm);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
}
|
|
|
|
|
|
2015-12-29 20:39:39 +01:00
|
|
|
|
void
|
2016-01-04 22:40:14 +01:00
|
|
|
|
kore_pool_cleanup(struct kore_pool *pool)
|
2015-12-29 20:39:39 +01:00
|
|
|
|
{
|
2023-01-05 22:47:29 +01:00
|
|
|
|
struct kore_pool_entry *entry, *next;
|
|
|
|
|
|
|
|
|
|
if (kore_mem_guard) {
|
|
|
|
|
for (entry = pool->freelist; entry != NULL; entry = next) {
|
|
|
|
|
pool_mark_entry_rw(pool, entry);
|
|
|
|
|
next = entry->nextfree;
|
|
|
|
|
(void)munmap(entry, pool->memsz);
|
|
|
|
|
}
|
|
|
|
|
}
|
2015-12-29 20:39:39 +01:00
|
|
|
|
|
2018-02-14 13:48:49 +01:00
|
|
|
|
free(pool->name);
|
2015-12-29 20:39:39 +01:00
|
|
|
|
}
|
|
|
|
|
|
2013-07-15 10:13:36 +02:00
|
|
|
|
void *
|
|
|
|
|
kore_pool_get(struct kore_pool *pool)
|
|
|
|
|
{
|
2023-01-05 22:47:29 +01:00
|
|
|
|
u_int64_t canary;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
struct kore_pool_entry *entry;
|
|
|
|
|
|
2016-07-12 16:29:30 +02:00
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
pool_lock(pool);
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (pool->freelist == NULL)
|
|
|
|
|
pool_grow(pool, pool->growth);
|
|
|
|
|
|
|
|
|
|
entry = pool->freelist;
|
|
|
|
|
|
|
|
|
|
if (kore_mem_guard)
|
|
|
|
|
pool_mark_entry_rw(pool, entry);
|
|
|
|
|
|
|
|
|
|
pool->freelist = entry->nextfree;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2014-03-06 08:20:05 +01:00
|
|
|
|
if (entry->state != POOL_ELEMENT_FREE)
|
2022-08-18 15:20:55 +02:00
|
|
|
|
fatal("%s: element %p was not free", pool->name, (void *)entry);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
entry->nextfree = NULL;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
entry->state = POOL_ELEMENT_BUSY;
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
canary = pool->canary;
|
|
|
|
|
canary ^= (uintptr_t)entry;
|
|
|
|
|
canary ^= (uintptr_t)entry->uptr;
|
|
|
|
|
|
|
|
|
|
memcpy(entry->canary, &canary, sizeof(canary));
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2016-07-12 16:29:30 +02:00
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
pool_unlock(pool);
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
return (entry->uptr);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
kore_pool_put(struct kore_pool *pool, void *ptr)
|
|
|
|
|
{
|
2023-01-05 22:47:29 +01:00
|
|
|
|
void *base;
|
|
|
|
|
u_int64_t canary;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
struct kore_pool_entry *entry;
|
|
|
|
|
|
2016-07-12 16:29:30 +02:00
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
pool_lock(pool);
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (kore_mem_guard) {
|
|
|
|
|
base = (u_int8_t *)ptr - ((uintptr_t)ptr % pool->pagesz);
|
|
|
|
|
} else {
|
|
|
|
|
base = (u_int8_t *)ptr - sizeof(*entry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
entry = (struct kore_pool_entry *)base;
|
|
|
|
|
|
|
|
|
|
if (entry->uptr != ptr) {
|
|
|
|
|
fatal("%s: uptr mismatch %p != %p",
|
|
|
|
|
pool->name, entry->uptr, ptr);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
memcpy(&canary, entry->canary, sizeof(canary));
|
|
|
|
|
canary ^= (uintptr_t)entry;
|
|
|
|
|
canary ^= (uintptr_t)ptr;
|
|
|
|
|
|
|
|
|
|
if (canary != pool->canary)
|
|
|
|
|
fatal("%s: memory corruption detected", pool->name);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
|
|
|
|
if (entry->state != POOL_ELEMENT_BUSY)
|
|
|
|
|
fatal("%s: element %p was not busy", pool->name, ptr);
|
|
|
|
|
|
|
|
|
|
entry->state = POOL_ELEMENT_FREE;
|
2023-01-05 22:47:29 +01:00
|
|
|
|
entry->nextfree = pool->freelist;
|
|
|
|
|
|
|
|
|
|
if (kore_mem_guard)
|
|
|
|
|
pool_mark_entry_none(pool, entry);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool->freelist = entry;
|
2016-07-12 16:29:30 +02:00
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
pool_unlock(pool);
|
|
|
|
|
#endif
|
2013-07-15 10:13:36 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool_grow(struct kore_pool *pool, size_t elms)
|
2013-07-15 10:13:36 +02:00
|
|
|
|
{
|
2016-07-12 14:01:02 +02:00
|
|
|
|
size_t i;
|
2023-01-05 22:47:29 +01:00
|
|
|
|
u_int8_t *base, *p;
|
|
|
|
|
struct kore_pool_entry *entry, *prev;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
prev = pool->freelist;
|
2016-07-12 13:54:14 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (kore_mem_guard == 0)
|
|
|
|
|
base = kore_mmap_region(elms * (sizeof(*entry) + pool->elmlen));
|
|
|
|
|
else
|
|
|
|
|
base = NULL;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
for (i = 0; i < elms; i++) {
|
|
|
|
|
if (kore_mem_guard) {
|
|
|
|
|
base = kore_mmap_region(pool->memsz);
|
|
|
|
|
p = base + (pool->memsz - pool->pagesz - pool->elmlen);
|
|
|
|
|
entry = (struct kore_pool_entry *)base;
|
|
|
|
|
} else {
|
|
|
|
|
p = base + ((sizeof(*entry) + pool->elmlen) * i);
|
|
|
|
|
entry = (struct kore_pool_entry *)p;
|
|
|
|
|
p += sizeof(*entry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
entry->uptr = p;
|
|
|
|
|
entry->nextfree = NULL;
|
|
|
|
|
entry->state = POOL_ELEMENT_FREE;
|
|
|
|
|
entry->canary = p + pool->uselen;
|
2016-07-12 13:54:14 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (prev != NULL) {
|
|
|
|
|
prev->nextfree = entry;
|
|
|
|
|
if (kore_mem_guard)
|
|
|
|
|
pool_mark_entry_none(pool, prev);
|
|
|
|
|
}
|
2016-07-12 13:54:14 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
prev = entry;
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (pool->freelist == NULL)
|
|
|
|
|
pool->freelist = entry;
|
|
|
|
|
|
|
|
|
|
if (kore_mem_guard) {
|
|
|
|
|
p += pool->elmlen;
|
|
|
|
|
|
|
|
|
|
if (((uintptr_t)p % pool->pagesz) != 0)
|
|
|
|
|
fatal("%s: misaligned page", __func__);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (mprotect(p, pool->pagesz, PROT_NONE) == -1)
|
|
|
|
|
fatal("%s: mprotect: %s", __func__, errno_s);
|
|
|
|
|
|
|
|
|
|
if (madvise(p, pool->pagesz, MADV_FREE) == -1)
|
|
|
|
|
fatal("%s: madvise: %s", __func__, errno_s);
|
|
|
|
|
}
|
2013-07-15 10:13:36 +02:00
|
|
|
|
}
|
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (prev != NULL && kore_mem_guard)
|
|
|
|
|
pool_mark_entry_none(pool, prev);
|
2013-07-15 10:13:36 +02:00
|
|
|
|
}
|
2015-12-29 20:39:39 +01:00
|
|
|
|
|
|
|
|
|
static void
|
2023-01-05 22:47:29 +01:00
|
|
|
|
pool_mark_entry_none(struct kore_pool *pool, void *ptr)
|
2015-12-29 20:39:39 +01:00
|
|
|
|
{
|
2023-01-05 22:47:29 +01:00
|
|
|
|
if (mprotect(ptr, pool->memsz - pool->pagesz, PROT_NONE) == -1)
|
|
|
|
|
fatal("%s: mprotect: %s", __func__, errno_s);
|
|
|
|
|
}
|
2015-12-29 20:39:39 +01:00
|
|
|
|
|
2023-01-05 22:47:29 +01:00
|
|
|
|
static void
|
|
|
|
|
pool_mark_entry_rw(struct kore_pool *pool, void *ptr)
|
|
|
|
|
{
|
|
|
|
|
if (mprotect(ptr, pool->memsz - pool->pagesz,
|
|
|
|
|
PROT_READ | PROT_WRITE) == -1)
|
|
|
|
|
fatal("%s: mprotect: %s", __func__, errno_s);
|
2015-12-29 20:39:39 +01:00
|
|
|
|
}
|
2016-07-12 16:29:30 +02:00
|
|
|
|
|
|
|
|
|
#if defined(KORE_USE_TASKS)
|
|
|
|
|
static void
|
|
|
|
|
pool_lock(struct kore_pool *pool)
|
|
|
|
|
{
|
|
|
|
|
for (;;) {
|
|
|
|
|
if (__sync_bool_compare_and_swap(&pool->lock, 0, 1))
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
pool_unlock(struct kore_pool *pool)
|
|
|
|
|
{
|
|
|
|
|
if (!__sync_bool_compare_and_swap(&pool->lock, 1, 0))
|
|
|
|
|
fatal("pool_unlock: failed to release %s", pool->name);
|
|
|
|
|
}
|
|
|
|
|
#endif
|