gcc/libcilkrts/runtime/cilk_fiber.cpp

1079 lines
32 KiB
C++
Raw Normal View History

/* cilk_fiber.cpp -*-C++-*-
*
*************************************************************************
*
* @copyright
* Copyright (C) 2012-2013, Intel Corporation
* All rights reserved.
*
* @copyright
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* @copyright
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
* WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
**************************************************************************/
/* Implementations of non-platform-specific aspects of cilk_fiber, especially
* the cilk_fiber_pool interface.
*/
#include "cilk_fiber.h"
#ifdef _WIN32
# include "cilk_fiber-win.h"
#else
# include "cilk_fiber-unix.h"
#endif
#include "cilk_malloc.h"
#include "bug.h"
#include <new>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "sysdep.h"
extern "C" {
inline int cilk_fiber_pool_sanity_check(cilk_fiber_pool *pool, const char* desc)
{
int errors = 0;
#if FIBER_DEBUG >= 1
if ((NULL != pool) && pool->total > 0) {
// Root pool should not allocate more fibers than alloc_max
errors += ((pool->parent == NULL) &&
(pool->total > pool->alloc_max));
errors += (pool->total > pool->high_water);
if (errors) {
fprintf(stderr, "ERROR at %s: pool=%p has max_size=%u, total=%d, high_water=%d\n",
desc,
pool, pool->max_size, pool->total, pool->high_water);
}
}
#endif
return (errors == 0);
}
inline void increment_pool_total(cilk_fiber_pool* pool)
{
++pool->total;
if (pool->high_water < pool->total)
pool->high_water = pool->total;
}
inline void decrement_pool_total(cilk_fiber_pool* pool, int fibers_freed)
{
pool->total -= fibers_freed;
}
/**
* @brief Free fibers from this pool until we have at most @c
* num_to_keep fibers remaining, and then put a fiber back.
*
* @pre We do not hold @c pool->lock
* @post After completion, we do not hold @c pool->lock
*/
static void cilk_fiber_pool_free_fibers_from_pool(cilk_fiber_pool* pool,
unsigned num_to_keep,
cilk_fiber* fiber_to_return)
{
// Free our own fibers, until we fall below our desired threshold.
// Each iteration of this loop proceeds in the following stages:
// 1. Acquire the pool lock,
// 2. Grabs up to B fibers from the pool, stores them into a buffer.
// 3. Check if pool is empty enough. If yes, put the last fiber back,
// and remember that we should quit.
// 4. Release the pool lock, and actually free any buffered fibers.
// 5. Check if we are done and should exit the loop. Otherwise, try again.
//
const bool need_lock = pool->lock;
bool last_fiber_returned = false;
do {
const int B = 10; // Pull at most this many fibers from the
// parent for one lock acquisition. Make
// this value large enough to amortize
// against the cost of acquiring and
// releasing the lock.
int num_to_free = 0;
cilk_fiber* fibers_to_free[B];
// Stage 1: Grab the lock.
if (need_lock) {
spin_mutex_lock(pool->lock);
}
// Stage 2: Grab up to B fibers to free.
int fibers_freed = 0;
while ((pool->size > num_to_keep) && (num_to_free < B)) {
fibers_to_free[num_to_free++] = pool->fibers[--pool->size];
fibers_freed++;
}
decrement_pool_total(pool, fibers_freed);
// Stage 3. Pool is below threshold. Put extra fiber back.
if (pool->size <= num_to_keep) {
// Put the last fiber back into the pool.
if (fiber_to_return) {
CILK_ASSERT(pool->size < pool->max_size);
pool->fibers[pool->size] = fiber_to_return;
pool->size++;
}
last_fiber_returned = true;
}
// Stage 4: Release the lock, and actually free any fibers
// buffered.
if (need_lock) {
spin_mutex_unlock(pool->lock);
}
for (int i = 0; i < num_to_free; ++i) {
fibers_to_free[i]->deallocate_to_heap();
}
} while (!last_fiber_returned);
}
/******************************************************************
* TBD: We want to simplify / rework the logic for allocating and
* deallocating fibers, so that they are hopefully simpler and work
* more elegantly for more than two levels.
******************************************************************/
/**
* @brief Transfer fibers from @c pool to @c pool->parent.
*
* @pre Must hold @c pool->lock if it exists.
* @post After completion, some number of fibers
* have been moved from this pool to the parent.
* The lock @c pool->lock is still held.
*
* TBD: Do we wish to guarantee that the lock has never been
* released? It may depend on the implementation...
*/
static void cilk_fiber_pool_move_fibers_to_parent_pool(cilk_fiber_pool* pool,
unsigned num_to_keep)
{
// ASSERT: We should hold the lock on pool (if it has one).
CILK_ASSERT(pool->parent);
cilk_fiber_pool* parent_pool = pool->parent;
// Move fibers from our pool to the parent until we either run out
// of space in the parent, or hit our threshold.
//
// This operation must be done while holding the parent lock.
// If the parent pool appears to be full, just return early.
if (parent_pool->size >= parent_pool->max_size)
return;
spin_mutex_lock(pool->parent->lock);
while ((parent_pool->size < parent_pool->max_size) &&
(pool->size > num_to_keep)) {
parent_pool->fibers[parent_pool->size++] =
pool->fibers[--pool->size];
}
// If the child pool has deallocated more than fibers to the heap
// than it has allocated, then transfer this "surplus" to the
// parent, so that the parent is free to allocate more from the
// heap.
//
// This transfer means that the total in the parent can
// temporarily go negative.
if (pool->total < 0) {
// Reduce parent total by the surplus we have in the local
// pool.
parent_pool->total += pool->total;
pool->total = 0;
}
spin_mutex_unlock(pool->parent->lock);
}
void cilk_fiber_pool_init(cilk_fiber_pool* pool,
cilk_fiber_pool* parent,
size_t stack_size,
unsigned buffer_size,
int alloc_max,
int is_shared)
{
#if FIBER_DEBUG >= 1
fprintf(stderr, "fiber_pool_init, pool=%p, parent=%p, alloc_max=%u\n",
pool, parent, alloc_max);
#endif
pool->lock = (is_shared ? spin_mutex_create() : NULL);
pool->parent = parent;
pool->stack_size = stack_size;
pool->max_size = buffer_size;
pool->size = 0;
pool->total = 0;
pool->high_water = 0;
pool->alloc_max = alloc_max;
pool->fibers =
(cilk_fiber**) __cilkrts_malloc(buffer_size * sizeof(cilk_fiber*));
CILK_ASSERT(NULL != pool->fibers);
#ifdef __MIC__
#define PREALLOCATE_FIBERS
#endif
#ifdef PREALLOCATE_FIBERS
// Pre-allocate 1/4 of fibers in the pools ahead of time. This
// value is somewhat arbitrary. It was chosen to be less than the
// threshold (of about 3/4) of fibers to keep in the pool when
// transferring fibers to the parent.
int pre_allocate_count = buffer_size/4;
for (pool->size = 0; pool->size < pre_allocate_count; pool->size++) {
pool->fibers[pool->size] = cilk_fiber::allocate_from_heap(pool->stack_size);
}
#endif
}
void cilk_fiber_pool_set_fiber_limit(cilk_fiber_pool* root_pool,
unsigned max_fibers_to_allocate)
{
// Should only set limit on root pool, not children.
CILK_ASSERT(NULL == root_pool->parent);
root_pool->alloc_max = max_fibers_to_allocate;
}
void cilk_fiber_pool_destroy(cilk_fiber_pool* pool)
{
CILK_ASSERT(cilk_fiber_pool_sanity_check(pool, "pool_destroy"));
// Lock my own pool, if I need to.
if (pool->lock) {
spin_mutex_lock(pool->lock);
}
// Give any remaining fibers to parent pool.
if (pool->parent) {
cilk_fiber_pool_move_fibers_to_parent_pool(pool, 0);
}
// Unlock pool.
if (pool->lock) {
spin_mutex_unlock(pool->lock);
}
// If I have any left in my pool, just free them myself.
// This method may acquire the pool lock.
cilk_fiber_pool_free_fibers_from_pool(pool, 0, NULL);
// Destroy the lock if there is one.
if (pool->lock) {
spin_mutex_destroy(pool->lock);
}
__cilkrts_free(pool->fibers);
}
cilk_fiber* cilk_fiber_allocate(cilk_fiber_pool* pool)
{
CILK_ASSERT(cilk_fiber_pool_sanity_check(pool, "allocate"));
return cilk_fiber::allocate(pool);
}
cilk_fiber* cilk_fiber_allocate_from_heap(size_t stack_size)
{
return cilk_fiber::allocate_from_heap(stack_size);
}
void cilk_fiber_reset_state(cilk_fiber* fiber, cilk_fiber_proc start_proc)
{
fiber->reset_state(start_proc);
}
int cilk_fiber_remove_reference(cilk_fiber *fiber, cilk_fiber_pool *pool)
{
return fiber->remove_reference(pool);
}
cilk_fiber* cilk_fiber_allocate_from_thread()
{
return cilk_fiber::allocate_from_thread();
}
int cilk_fiber_deallocate_from_thread(cilk_fiber *fiber)
{
return fiber->deallocate_from_thread();
}
int cilk_fiber_remove_reference_from_thread(cilk_fiber *fiber)
{
return fiber->remove_reference_from_thread();
}
int cilk_fiber_is_allocated_from_thread(cilk_fiber *fiber)
{
return fiber->is_allocated_from_thread();
}
#if SUPPORT_GET_CURRENT_FIBER
cilk_fiber* cilk_fiber_get_current_fiber(void)
{
return cilk_fiber::get_current_fiber();
}
#endif
void cilk_fiber_suspend_self_and_resume_other(cilk_fiber* self,
cilk_fiber* other)
{
self->suspend_self_and_resume_other(other);
}
void cilk_fiber::reset_state(cilk_fiber_proc start_proc)
{
// Setup the fiber and return.
this->m_start_proc = start_proc;
CILK_ASSERT(!this->is_resumable());
CILK_ASSERT(NULL == this->m_pending_remove_ref);
CILK_ASSERT(NULL == this->m_pending_pool);
}
NORETURN
cilk_fiber_remove_reference_from_self_and_resume_other(cilk_fiber* self,
cilk_fiber_pool* self_pool,
cilk_fiber* other)
{
#if FIBER_DEBUG >= 3
__cilkrts_worker* w = __cilkrts_get_tls_worker();
fprintf(stderr, "W=%d: cilk_fiber_deactivate_self_and_resume_other: self=%p, other=%p\n",
w->self,
self, other);
#endif
CILK_ASSERT(cilk_fiber_pool_sanity_check(self_pool, "remove_reference_from_self_resume_other"));
self->remove_reference_from_self_and_resume_other(self_pool, other);
// We should never return here.
}
void cilk_fiber_set_post_switch_proc(cilk_fiber *self,
cilk_fiber_proc post_switch_proc)
{
self->set_post_switch_proc(post_switch_proc);
}
void cilk_fiber_invoke_tbb_stack_op(cilk_fiber* fiber,
__cilk_tbb_stack_op op)
{
fiber->invoke_tbb_stack_op(op);
}
cilk_fiber_data* cilk_fiber_get_data(cilk_fiber* fiber)
{
return fiber->get_data();
/// TBD: Change this code to "return (cilk_fiber_data*)fiber;"
// plus a static assert, so that this function is
// more easily inlined by the compiler.
}
int cilk_fiber_is_resumable(cilk_fiber *fiber)
{
return fiber->is_resumable();
}
char* cilk_fiber_get_stack_base(cilk_fiber *fiber)
{
return fiber->get_stack_base();
}
#if defined(_WIN32) && 0 // Only works on Windows. Disable debugging for now.
#define DBG_STACK_OPS(_fmt, ...) __cilkrts_dbgprintf(_fmt, __VA_ARGS__)
#else
#define DBG_STACK_OPS(_fmt, ...)
#endif
void cilk_fiber_set_stack_op(cilk_fiber *fiber,
__cilk_tbb_stack_op_thunk o)
{
cilk_fiber_data *fdata = cilk_fiber_get_data(fiber);
DBG_STACK_OPS ("cilk_fiber_set_stack_op - cilk_fiber %p, routine: %p, data: %p\n",
fiber,
o.routine,
o.data);
fdata->stack_op_routine = o.routine;
fdata->stack_op_data = o.data;
}
#if 0 // Debugging function
static
const char *NameStackOp (enum __cilk_tbb_stack_op op)
{
switch(op)
{
case CILK_TBB_STACK_ORPHAN: return "CILK_TBB_STACK_ORPHAN";
case CILK_TBB_STACK_ADOPT: return "CILK_TBB_STACK_ADOPT";
case CILK_TBB_STACK_RELEASE: return "CILK_TBB_STACK_RELEASE";
default: return "Unknown";
}
}
#endif
/*
* Save TBB interop information for an unbound thread. It will get picked
* up when the thread is bound to the runtime.
*/
void cilk_fiber_tbb_interop_save_stack_op_info(__cilk_tbb_stack_op_thunk o)
{
__cilk_tbb_stack_op_thunk *saved_thunk =
__cilkrts_get_tls_tbb_interop();
DBG_STACK_OPS("Calling save_stack_op; o.routine=%p, o.data=%p, saved_thunk=%p\n",
o.routine, o.data, saved_thunk);
// If there is not already space allocated, allocate some.
if (NULL == saved_thunk) {
saved_thunk = (__cilk_tbb_stack_op_thunk*)
__cilkrts_malloc(sizeof(__cilk_tbb_stack_op_thunk));
__cilkrts_set_tls_tbb_interop(saved_thunk);
}
*saved_thunk = o;
DBG_STACK_OPS ("Unbound Thread %04x: tbb_interop_save_stack_op_info - saved info\n",
cilkos_get_current_thread_id());
}
/*
* Save TBB interop information from the cilk_fiber. It will get picked
* up when the thread is bound to the runtime next time.
*/
void cilk_fiber_tbb_interop_save_info_from_stack(cilk_fiber *fiber)
{
__cilk_tbb_stack_op_thunk *saved_thunk;
cilk_fiber_data* fdata;
if (NULL == fiber)
return;
fdata = cilk_fiber_get_data(fiber);
// If there is no TBB interop data, just return
if (NULL == fdata->stack_op_routine)
return;
saved_thunk = __cilkrts_get_tls_tbb_interop();
// If there is not already space allocated, allocate some.
if (NULL == saved_thunk) {
saved_thunk = (__cilk_tbb_stack_op_thunk*)
__cilkrts_malloc(sizeof(__cilk_tbb_stack_op_thunk));
__cilkrts_set_tls_tbb_interop(saved_thunk);
}
saved_thunk->routine = fdata->stack_op_routine;
saved_thunk->data = fdata->stack_op_data;
}
/*
* If there's TBB interop information that was saved before the thread was
* bound, apply it now
*/
void cilk_fiber_tbb_interop_use_saved_stack_op_info(cilk_fiber* fiber)
{
__cilk_tbb_stack_op_thunk *saved_thunk =
__cilkrts_get_tls_tbb_interop();
CILK_ASSERT(fiber);
// If we haven't allocated a TBB interop index, we don't have any saved info
if (NULL == saved_thunk) {
DBG_STACK_OPS ("cilk_fiber %p: tbb_interop_use_saved_stack_op_info - no saved info\n",
fiber);
return;
}
DBG_STACK_OPS ("cilk_fiber %p: tbb_interop_use_saved_stack_op_info - using saved info\n",
fiber);
// Associate the saved info with the __cilkrts_stack
cilk_fiber_set_stack_op(fiber, *saved_thunk);
// Free the saved data. We'll save it again if needed when the code
// returns from the initial function
cilk_fiber_tbb_interop_free_stack_op_info();
}
/*
* Free saved TBB interop memory. Should only be called when the thread is
* not bound.
*/
void cilk_fiber_tbb_interop_free_stack_op_info(void)
{
__cilk_tbb_stack_op_thunk *saved_thunk =
__cilkrts_get_tls_tbb_interop();
// If we haven't allocated a TBB interop index, we don't have any saved info
if (NULL == saved_thunk)
return;
DBG_STACK_OPS ("tbb_interop_free_stack_op_info - freeing saved info\n");
// Free the memory and wipe out the TLS value
__cilkrts_free(saved_thunk);
__cilkrts_set_tls_tbb_interop(NULL);
}
#if NEED_FIBER_REF_COUNTS
int cilk_fiber_has_references(cilk_fiber *fiber)
{
return (fiber->get_ref_count() > 0);
}
int cilk_fiber_get_ref_count(cilk_fiber *fiber)
{
return fiber->get_ref_count();
}
void cilk_fiber_add_reference(cilk_fiber *fiber)
{
fiber->inc_ref_count();
}
#endif // NEED_FIBER_REF_COUNTS
} // End extern "C"
cilk_fiber_sysdep* cilk_fiber::sysdep()
{
return static_cast<cilk_fiber_sysdep*>(this);
}
cilk_fiber::cilk_fiber()
: m_start_proc(NULL)
, m_post_switch_proc(NULL)
, m_pending_remove_ref(NULL)
, m_pending_pool(NULL)
, m_flags(0)
{
// Clear cilk_fiber_data base-class data members
std::memset((cilk_fiber_data*) this, 0, sizeof(cilk_fiber_data));
// cilk_fiber data members
init_ref_count(0);
}
cilk_fiber::cilk_fiber(std::size_t stack_size)
{
*this = cilk_fiber(); // A delegating constructor would be nice here
this->stack_size = stack_size;
}
cilk_fiber::~cilk_fiber()
{
// Empty destructor.
}
char* cilk_fiber::get_stack_base()
{
return this->sysdep()->get_stack_base_sysdep();
}
cilk_fiber* cilk_fiber::allocate_from_heap(std::size_t stack_size)
{
// Case 1: pool is NULL. create a new fiber from the heap
// No need for locks here.
cilk_fiber_sysdep* ret =
(cilk_fiber_sysdep*) __cilkrts_malloc(sizeof(cilk_fiber_sysdep));
// Error condition. If we failed to allocate a fiber from the
// heap, we are in trouble though...
if (!ret)
return NULL;
::new(ret) cilk_fiber_sysdep(stack_size);
CILK_ASSERT(0 == ret->m_flags);
CILK_ASSERT(NULL == ret->m_pending_remove_ref);
CILK_ASSERT(NULL == ret->m_pending_pool);
ret->init_ref_count(1);
return ret;
}
#if USE_FIBER_TRY_ALLOCATE_FROM_POOL
/**
* Helper method: try to allocate a fiber from this pool or its
* ancestors without going to the OS / heap.
*
* Returns allocated pool, or NULL if no pool is found.
*
* If pool contains a suitable fiber. Return it. Otherwise, try to
* recursively grab a fiber from the parent pool, if there is one.
*
* This method will not allocate a fiber from the heap.
*
* This method could be written either recursively or iteratively.
* It probably does not matter which one we do.
*
* @note This method is compiled, but may not be used unless the
* USE_FIBER_TRY_ALLOCATE_FROM_POOL switch is set.
*/
cilk_fiber* cilk_fiber::try_allocate_from_pool_recursive(cilk_fiber_pool* pool)
{
cilk_fiber* ret = NULL;
if (pool->size > 0) {
// Try to get the lock.
if (pool->lock) {
// For some reason, it seems to be better to just block on the parent
// pool lock, instead of using a try-lock?
#define USE_TRY_LOCK_IN_FAST_ALLOCATE 0
#if USE_TRY_LOCK_IN_FAST_ALLOCATE
int got_lock = spin_mutex_trylock(pool->lock);
if (!got_lock) {
// If we fail, skip to the parent.
if (pool->parent) {
return try_allocate_from_pool_recursive(pool->parent);
}
}
#else
spin_mutex_lock(pool->lock);
#endif
}
// Check in the pool if we have the lock.
if (pool->size > 0) {
ret = pool->fibers[--pool->size];
}
// Release the lock once we are done updating pool fields.
if (pool->lock) {
spin_mutex_unlock(pool->lock);
}
}
if ((!ret) && (pool->parent)) {
return try_allocate_from_pool_recursive(pool->parent);
}
if (ret) {
// When we pull a fiber out of the pool, set its reference
// count before we return it.
ret->init_ref_count(1);
}
return ret;
}
#endif // USE_FIBER_TRY_ALLOCATE_FROM_POOL
cilk_fiber* cilk_fiber::allocate(cilk_fiber_pool* pool)
{
// Pool should not be NULL in this method. But I'm not going to
// actually assert it, because we are likely to seg fault anyway
// if it is.
// CILK_ASSERT(NULL != pool);
cilk_fiber *ret = NULL;
#if USE_FIBER_TRY_ALLOCATE_FROM_POOL
// "Fast" path, which doesn't go to the heap or OS until checking
// the ancestors first.
ret = try_allocate_from_pool_recursive(pool);
if (ret)
return ret;
#endif
// If we don't get anything from the "fast path", then go through
// a slower path to look for a fiber.
//
// 1. Lock the pool if it is shared.
// 2. Look in our local pool. If we find one, release the lock
// and quit searching.
// 3. Otherwise, check whether we can allocate from heap.
// 4. Release the lock if it was acquired.
// 5. Try to allocate from the heap, if step 3 said we could.
// If we find a fiber, then quit searching.
// 6. If none of these steps work, just recursively try again
// from the parent.
// 1. Lock the pool if it is shared.
if (pool->lock) {
spin_mutex_lock(pool->lock);
}
// 2. Look in local pool.
if (pool->size > 0) {
ret = pool->fibers[--pool->size];
if (ret) {
// If we found one, release the lock once we are
// done updating pool fields, and break out of the
// loop.
if (pool->lock) {
spin_mutex_unlock(pool->lock);
}
// When we pull a fiber out of the pool, set its reference
// count just in case.
ret->init_ref_count(1);
return ret;
}
}
// 3. Check whether we can allocate from the heap.
bool can_allocate_from_heap = false;
if (pool->total < pool->alloc_max) {
// Track that we are allocating a new fiber from the
// heap, originating from this pool.
// This increment may be undone if we happen to fail to
// allocate from the heap.
increment_pool_total(pool);
can_allocate_from_heap = true;
}
// 4. Unlock the pool, and then allocate from the heap.
if (pool->lock) {
spin_mutex_unlock(pool->lock);
}
// 5. Actually try to allocate from the heap / OS.
if (can_allocate_from_heap) {
ret = allocate_from_heap(pool->stack_size);
// If we got something from the heap, just return it.
if (ret) {
return ret;
}
// Otherwise, we failed in our attempt to allocate a
// fiber from the heap. Grab the lock and decrement
// the total again.
if (pool->lock) {
spin_mutex_lock(pool->lock);
}
decrement_pool_total(pool, 1);
if (pool->lock) {
spin_mutex_unlock(pool->lock);
}
}
// 6. If we get here, then searching this pool failed. Go search
// the parent instead if we have one.
if (pool->parent) {
return allocate(pool->parent);
}
return ret;
}
int cilk_fiber::remove_reference(cilk_fiber_pool* pool)
{
int ref_count = this->dec_ref_count();
if (ref_count == 0) {
if (pool) {
deallocate_self(pool);
}
else {
deallocate_to_heap();
}
}
return ref_count;
}
cilk_fiber* cilk_fiber::allocate_from_thread()
{
void* retmem = __cilkrts_malloc(sizeof(cilk_fiber_sysdep));
CILK_ASSERT(retmem);
cilk_fiber_sysdep* ret = ::new(retmem) cilk_fiber_sysdep(from_thread);
// A fiber allocated from a thread begins with a reference count
// of 2. The first is for being created, and the second is for
// being running.
//
// Suspending this fiber will decrement the count down to 1.
ret->init_ref_count(2);
#if SUPPORT_GET_CURRENT_FIBER
// We're creating the main fiber for this thread. Set this fiber as the
// current fiber.
cilkos_set_tls_cilk_fiber(ret);
#endif
return ret;
}
int cilk_fiber::deallocate_from_thread()
{
CILK_ASSERT(this->is_allocated_from_thread());
#if SUPPORT_GET_CURRENT_FIBER
CILK_ASSERT(this == cilkos_get_tls_cilk_fiber());
// Reverse of "allocate_from_thread".
cilkos_set_tls_cilk_fiber(NULL);
#endif
this->assert_ref_count_at_least(2);
// Suspending the fiber should conceptually decrement the ref
// count by 1.
cilk_fiber_sysdep* self = this->sysdep();
self->convert_fiber_back_to_thread();
// Then, freeing the fiber itself decrements the ref count again.
int ref_count = this->sub_from_ref_count(2);
if (ref_count == 0) {
self->~cilk_fiber_sysdep();
__cilkrts_free(self);
}
return ref_count;
}
int cilk_fiber::remove_reference_from_thread()
{
int ref_count = dec_ref_count();
if (ref_count == 0) {
cilk_fiber_sysdep* self = this->sysdep();
self->~cilk_fiber_sysdep();
__cilkrts_free(self);
}
return ref_count;
}
#if SUPPORT_GET_CURRENT_FIBER
cilk_fiber* cilk_fiber::get_current_fiber()
{
return cilk_fiber_sysdep::get_current_fiber_sysdep();
}
#endif
void cilk_fiber::do_post_switch_actions()
{
if (m_post_switch_proc)
{
cilk_fiber_proc proc = m_post_switch_proc;
m_post_switch_proc = NULL;
proc(this);
}
if (m_pending_remove_ref)
{
m_pending_remove_ref->remove_reference(m_pending_pool);
// Even if we don't free it,
m_pending_remove_ref = NULL;
m_pending_pool = NULL;
}
}
void cilk_fiber::suspend_self_and_resume_other(cilk_fiber* other)
{
#if FIBER_DEBUG >=1
fprintf(stderr, "suspend_self_and_resume_other: self =%p, other=%p [owner=%p, resume_sf=%p]\n",
this, other, other->owner, other->resume_sf);
#endif
// Decrement my reference count (to suspend)
// Increment other's count (to resume)
// Suspended fiber should have a reference count of at least 1. (It is not in a pool).
this->dec_ref_count();
other->inc_ref_count();
this->assert_ref_count_at_least(1);
// Pass along my owner.
other->owner = this->owner;
this->owner = NULL;
// Change this fiber to resumable.
CILK_ASSERT(!this->is_resumable());
this->set_resumable(true);
// Normally, I'd assert other->is_resumable(). But this flag may
// be false the first time we try to "resume" a fiber.
cilk_fiber_sysdep* self = this->sysdep();
self->suspend_self_and_resume_other_sysdep(other->sysdep());
// HAVE RESUMED EXECUTION
// When we come back here, we should have at least two references:
// one for the fiber being allocated / out of a pool, and one for it being active.
this->assert_ref_count_at_least(2);
}
NORETURN
cilk_fiber::remove_reference_from_self_and_resume_other(cilk_fiber_pool* self_pool,
cilk_fiber* other)
{
// Decrement my reference count once (to suspend)
// Increment other's count (to resume)
// Suspended fiber should have a reference count of at least 1. (It is not in a pool).
this->dec_ref_count();
other->inc_ref_count();
// Set a pending remove reference for this fiber, once we have
// actually switched off.
other->m_pending_remove_ref = this;
other->m_pending_pool = self_pool;
// Pass along my owner.
other->owner = this->owner;
this->owner = NULL;
// Since we are deallocating self, this fiber does not become
// resumable.
CILK_ASSERT(!this->is_resumable());
cilk_fiber_sysdep* self = this->sysdep();
self->jump_to_resume_other_sysdep(other->sysdep());
__cilkrts_bug("Deallocating fiber. We should never come back here.");
std::abort();
}
void cilk_fiber::deallocate_to_heap()
{
cilk_fiber_sysdep* self = this->sysdep();
self->~cilk_fiber_sysdep();
__cilkrts_free(self);
}
void cilk_fiber::deallocate_self(cilk_fiber_pool* pool)
{
this->set_resumable(false);
CILK_ASSERT(NULL != pool);
CILK_ASSERT(!this->is_allocated_from_thread());
this->assert_ref_count_equals(0);
// Cases:
//
// 1. pool has space: Add to this pool.
// 2. pool is full: Give some fibers to parent, and then free
// enough to make space for the fiber we are deallocating.
// Then put the fiber back into the pool.
const bool need_lock = pool->lock;
// Grab the lock for the remaining cases.
if (need_lock) {
spin_mutex_lock(pool->lock);
}
// Case 1: this pool has space. Return the fiber.
if (pool->size < pool->max_size)
{
// Add this fiber to pool
pool->fibers[pool->size++] = this;
if (need_lock) {
spin_mutex_unlock(pool->lock);
}
return;
}
// Case 2: Pool is full.
//
// First free up some space by giving fibers to the parent.
if (pool->parent)
{
// Pool is full. Move all but "num_to_keep" fibers to parent,
// if we can.
unsigned num_to_keep = pool->max_size/2 + pool->max_size/4;
cilk_fiber_pool_move_fibers_to_parent_pool(pool, num_to_keep);
}
if (need_lock) {
spin_mutex_unlock(pool->lock);
}
// Now, free a fiber to make room for the one we need to put back,
// and then put this fiber back. This step may actually return
// fibers to the heap.
cilk_fiber_pool_free_fibers_from_pool(pool, pool->max_size -1, this);
}
// NOTE: Except for print-debug, this code is the same as in Windows.
void cilk_fiber::invoke_tbb_stack_op(__cilk_tbb_stack_op op)
{
cilk_fiber_data *fdata = this->get_data();
if (0 == fdata->stack_op_routine)
{
if (CILK_TBB_STACK_RELEASE != op)
DBG_STACK_OPS ("Wkr %p: invoke_tbb_stack_op - %s (%d) for cilk_fiber %p, fiber %p, thread id %04x - No stack op routine\n",
fdata->owner,
NameStackOp(op),
op,
fdata,
this,
cilkos_get_current_thread_id());
return;
}
// Call TBB to do it's thing
DBG_STACK_OPS ("Wkr %p: invoke_tbb_stack_op - op %s data %p for cilk_fiber %p, fiber %p, thread id %04x\n",
fdata->owner,
NameStackOp(op),
fdata->stack_op_data,
fdata,
this,
cilkos_get_current_thread_id());
(*fdata->stack_op_routine)(op, fdata->stack_op_data);
if (op == CILK_TBB_STACK_RELEASE)
{
fdata->stack_op_routine = 0;
fdata->stack_op_data = 0;
}
}
#if NEED_FIBER_REF_COUNTS
void cilk_fiber::atomic_inc_ref_count()
{
cilkos_atomic_add(&m_outstanding_references, 1);
}
long cilk_fiber::atomic_dec_ref_count()
{
return cilkos_atomic_add(&m_outstanding_references, -1);
}
long cilk_fiber::atomic_sub_from_ref_count(long v)
{
return cilkos_atomic_add(&m_outstanding_references, -v);
}
#endif // NEED_FIBER_REF_COUNTS
/* End cilk_fibers.cpp */