3038054c68
From-SVN: r204173
3941 lines
135 KiB
C
3941 lines
135 KiB
C
/* scheduler.c -*-C-*-
|
|
*
|
|
*************************************************************************
|
|
*
|
|
* @copyright
|
|
* Copyright (C) 2007-2013, Intel Corporation
|
|
* All rights reserved.
|
|
*
|
|
* @copyright
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
* contributors may be used to endorse or promote products derived
|
|
* from this software without specific prior written permission.
|
|
*
|
|
* @copyright
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
|
|
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
|
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
|
|
* WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
**************************************************************************/
|
|
|
|
/*
|
|
* Cilk scheduler
|
|
*/
|
|
|
|
#include "scheduler.h"
|
|
#include "bug.h"
|
|
#include "os.h"
|
|
#include "os_mutex.h"
|
|
#include "local_state.h"
|
|
#include "signal_node.h"
|
|
#include "full_frame.h"
|
|
#include "sysdep.h"
|
|
#include "except.h"
|
|
#include "cilk_malloc.h"
|
|
#include "pedigrees.h"
|
|
#include "record-replay.h"
|
|
|
|
#include <limits.h>
|
|
#include <string.h> /* memcpy */
|
|
#include <stdio.h> // sprintf
|
|
#include <stdlib.h> // malloc, free, abort
|
|
|
|
#ifdef _WIN32
|
|
# pragma warning(disable:1786) // disable warning: sprintf is deprecated
|
|
# include "sysdep-win.h"
|
|
# include "except-win32.h"
|
|
#endif // _WIN32
|
|
|
|
// ICL: Don't complain about conversion from pointer to same-sized integral
|
|
// type in __cilkrts_put_stack. That's why we're using ptrdiff_t
|
|
#ifdef _WIN32
|
|
# pragma warning(disable: 1684)
|
|
#endif
|
|
|
|
#include "cilk/cilk_api.h"
|
|
#include "frame_malloc.h"
|
|
#include "metacall_impl.h"
|
|
#include "reducer_impl.h"
|
|
#include "cilk-tbb-interop.h"
|
|
#include "cilk-ittnotify.h"
|
|
#include "stats.h"
|
|
|
|
// ICL: Don't complain about loss of precision in myrand
|
|
// I tried restoring the warning after the function, but it didn't
|
|
// suppress it
|
|
#ifdef _WIN32
|
|
# pragma warning(disable: 2259)
|
|
#endif
|
|
|
|
#ifndef _WIN32
|
|
# include <unistd.h>
|
|
#endif
|
|
|
|
#ifdef __VXWORKS__
|
|
// redeclare longjmp() with noreturn to stop warnings
|
|
extern __attribute__((noreturn))
|
|
void longjmp(jmp_buf, int);
|
|
#endif
|
|
|
|
//#define DEBUG_LOCKS 1
|
|
#ifdef DEBUG_LOCKS
|
|
// The currently executing worker must own this worker's lock
|
|
# define ASSERT_WORKER_LOCK_OWNED(w) \
|
|
{ \
|
|
__cilkrts_worker *tls_worker = __cilkrts_get_tls_worker(); \
|
|
CILK_ASSERT((w)->l->lock.owner == tls_worker); \
|
|
}
|
|
#else
|
|
# define ASSERT_WORKER_LOCK_OWNED(w)
|
|
#endif // DEBUG_LOCKS
|
|
|
|
// Options for the scheduler.
|
|
enum schedule_t { SCHEDULE_RUN,
|
|
SCHEDULE_WAIT,
|
|
SCHEDULE_EXIT };
|
|
|
|
// Return values for provably_good_steal()
|
|
enum provably_good_steal_t
|
|
{
|
|
ABANDON_EXECUTION, // Not the last child to the sync - attempt to steal work
|
|
CONTINUE_EXECUTION, // Last child to the sync - continue executing on this worker
|
|
WAIT_FOR_CONTINUE // The replay log indicates that this was the worker
|
|
// which continued. Loop until we are the last worker
|
|
// to the sync.
|
|
};
|
|
|
|
|
|
// Verify that "w" is the worker we are currently executing on.
|
|
// Because this check is expensive, this method is usually a no-op.
|
|
static inline void verify_current_wkr(__cilkrts_worker *w)
|
|
{
|
|
#if ((REDPAR_DEBUG >= 3) || (FIBER_DEBUG >= 1))
|
|
// Lookup the worker from TLS and compare to w.
|
|
__cilkrts_worker* tmp = __cilkrts_get_tls_worker();
|
|
if (w != tmp) {
|
|
fprintf(stderr, "Error. W=%d, actual worker =%d...\n",
|
|
w->self,
|
|
tmp->self);
|
|
}
|
|
CILK_ASSERT(w == tmp);
|
|
#endif
|
|
}
|
|
|
|
static enum schedule_t worker_runnable(__cilkrts_worker *w);
|
|
|
|
// Scheduling-fiber functions:
|
|
static void do_return_from_spawn (__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf);
|
|
static void do_sync (__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf);
|
|
|
|
// max is defined on Windows and VxWorks
|
|
#if (! defined(_WIN32)) && (! defined(__VXWORKS__))
|
|
// TBD: definition of max() for Linux.
|
|
# define max(a, b) ((a) < (b) ? (b) : (a))
|
|
#endif
|
|
|
|
void __cilkrts_dump_stats_to_stderr(global_state_t *g)
|
|
{
|
|
#ifdef CILK_PROFILE
|
|
int i;
|
|
for (i = 0; i < g->total_workers; ++i) {
|
|
// Print out statistics for each worker. We collected them,
|
|
// so why not print them out?
|
|
fprintf(stderr, "Stats for worker %d\n", i);
|
|
dump_stats_to_file(stderr, g->workers[i]->l->stats);
|
|
__cilkrts_accum_stats(&g->stats, g->workers[i]->l->stats);
|
|
}
|
|
|
|
// Also print out aggregate statistics.
|
|
dump_stats_to_file(stderr, &g->stats);
|
|
#endif
|
|
fprintf(stderr,
|
|
"CILK PLUS Thread Info: P=%d, Q=%d\n",
|
|
g->P,
|
|
g->Q);
|
|
fprintf(stderr,
|
|
"CILK PLUS RUNTIME MEMORY USAGE: %lld bytes",
|
|
(long long)g->frame_malloc.allocated_from_os);
|
|
#ifdef CILK_PROFILE
|
|
if (g->stats.stack_hwm)
|
|
fprintf(stderr, ", %ld stacks", g->stats.stack_hwm);
|
|
#endif
|
|
fputc('\n', stderr);
|
|
}
|
|
|
|
static void validate_worker(__cilkrts_worker *w)
|
|
{
|
|
/* check the magic numbers, for debugging purposes */
|
|
if (w->l->worker_magic_0 != WORKER_MAGIC_0 ||
|
|
w->l->worker_magic_1 != WORKER_MAGIC_1)
|
|
abort_because_rts_is_corrupted();
|
|
}
|
|
|
|
static void double_link(full_frame *left_ff, full_frame *right_ff)
|
|
{
|
|
if (left_ff)
|
|
left_ff->right_sibling = right_ff;
|
|
if (right_ff)
|
|
right_ff->left_sibling = left_ff;
|
|
}
|
|
|
|
/* add CHILD to the right of all children of PARENT */
|
|
static void push_child(full_frame *parent_ff, full_frame *child_ff)
|
|
{
|
|
double_link(parent_ff->rightmost_child, child_ff);
|
|
double_link(child_ff, 0);
|
|
parent_ff->rightmost_child = child_ff;
|
|
}
|
|
|
|
/* unlink CHILD from the list of all children of PARENT */
|
|
static void unlink_child(full_frame *parent_ff, full_frame *child_ff)
|
|
{
|
|
double_link(child_ff->left_sibling, child_ff->right_sibling);
|
|
|
|
if (!child_ff->right_sibling) {
|
|
/* this is the rightmost child -- update parent link */
|
|
CILK_ASSERT(parent_ff->rightmost_child == child_ff);
|
|
parent_ff->rightmost_child = child_ff->left_sibling;
|
|
}
|
|
child_ff->left_sibling = child_ff->right_sibling = 0; /* paranoia */
|
|
}
|
|
|
|
static void incjoin(full_frame *ff)
|
|
{
|
|
++ff->join_counter;
|
|
}
|
|
|
|
static int decjoin(full_frame *ff)
|
|
{
|
|
CILK_ASSERT(ff->join_counter > 0);
|
|
return (--ff->join_counter);
|
|
}
|
|
|
|
static int simulate_decjoin(full_frame *ff)
|
|
{
|
|
CILK_ASSERT(ff->join_counter > 0);
|
|
return (ff->join_counter - 1);
|
|
}
|
|
|
|
/*
|
|
* Pseudo-random generator defined by the congruence S' = 69070 * S
|
|
* mod (2^32 - 5). Marsaglia (CACM July 1993) says on page 107 that
|
|
* this is a ``good one''. There you go.
|
|
*
|
|
* The literature makes a big fuss about avoiding the division, but
|
|
* for us it is not worth the hassle.
|
|
*/
|
|
static const unsigned RNGMOD = ((1ULL << 32) - 5);
|
|
static const unsigned RNGMUL = 69070U;
|
|
|
|
static unsigned myrand(__cilkrts_worker *w)
|
|
{
|
|
unsigned state = w->l->rand_seed;
|
|
state = (unsigned)((RNGMUL * (unsigned long long)state) % RNGMOD);
|
|
w->l->rand_seed = state;
|
|
return state;
|
|
}
|
|
|
|
static void mysrand(__cilkrts_worker *w, unsigned seed)
|
|
{
|
|
seed %= RNGMOD;
|
|
seed += (seed == 0); /* 0 does not belong to the multiplicative
|
|
group. Use 1 instead */
|
|
w->l->rand_seed = seed;
|
|
}
|
|
|
|
/* W grabs its own lock */
|
|
void __cilkrts_worker_lock(__cilkrts_worker *w)
|
|
{
|
|
validate_worker(w);
|
|
CILK_ASSERT(w->l->do_not_steal == 0);
|
|
|
|
/* tell thieves to stay out of the way */
|
|
w->l->do_not_steal = 1;
|
|
__cilkrts_fence(); /* probably redundant */
|
|
|
|
__cilkrts_mutex_lock(w, &w->l->lock);
|
|
}
|
|
|
|
void __cilkrts_worker_unlock(__cilkrts_worker *w)
|
|
{
|
|
__cilkrts_mutex_unlock(w, &w->l->lock);
|
|
CILK_ASSERT(w->l->do_not_steal == 1);
|
|
/* The fence is probably redundant. Use a release
|
|
operation when supported (gcc and compatibile);
|
|
that is faster on x86 which serializes normal stores. */
|
|
#if defined __GNUC__ && (__GNUC__ * 10 + __GNUC_MINOR__ > 43 || __ICC >= 1110)
|
|
__sync_lock_release(&w->l->do_not_steal);
|
|
#else
|
|
w->l->do_not_steal = 0;
|
|
__cilkrts_fence(); /* store-store barrier, redundant on x86 */
|
|
#endif
|
|
}
|
|
|
|
/* try to acquire the lock of some *other* worker */
|
|
static int worker_trylock_other(__cilkrts_worker *w,
|
|
__cilkrts_worker *other)
|
|
{
|
|
int status = 0;
|
|
|
|
validate_worker(other);
|
|
|
|
/* This protocol guarantees that, after setting the DO_NOT_STEAL
|
|
flag, worker W can enter its critical section after waiting for
|
|
the thief currently in the critical section (if any) and at
|
|
most one other thief.
|
|
|
|
This requirement is overly paranoid, but it should protect us
|
|
against future nonsense from OS implementors.
|
|
*/
|
|
|
|
/* compete for the right to disturb OTHER */
|
|
if (__cilkrts_mutex_trylock(w, &other->l->steal_lock)) {
|
|
if (other->l->do_not_steal) {
|
|
/* leave it alone */
|
|
} else {
|
|
status = __cilkrts_mutex_trylock(w, &other->l->lock);
|
|
}
|
|
__cilkrts_mutex_unlock(w, &other->l->steal_lock);
|
|
}
|
|
|
|
|
|
return status;
|
|
}
|
|
|
|
static void worker_unlock_other(__cilkrts_worker *w,
|
|
__cilkrts_worker *other)
|
|
{
|
|
__cilkrts_mutex_unlock(w, &other->l->lock);
|
|
}
|
|
|
|
|
|
/* Lock macro Usage:
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
statement;
|
|
statement;
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
statement;
|
|
statement;
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
} END_WITH_WORKER_LOCK(w);
|
|
*/
|
|
#define BEGIN_WITH_WORKER_LOCK(w) __cilkrts_worker_lock(w); do
|
|
#define END_WITH_WORKER_LOCK(w) while (__cilkrts_worker_unlock(w), 0)
|
|
|
|
// TBD(jsukha): These are worker lock acquistions on
|
|
// a worker whose deque is empty. My conjecture is that we
|
|
// do not need to hold the worker lock at these points.
|
|
// I have left them in for now, however.
|
|
//
|
|
// #define REMOVE_POSSIBLY_OPTIONAL_LOCKS
|
|
#ifdef REMOVE_POSSIBLY_OPTIONAL_LOCKS
|
|
#define BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) do
|
|
#define END_WITH_WORKER_LOCK_OPTIONAL(w) while (0)
|
|
#else
|
|
#define BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) __cilkrts_worker_lock(w); do
|
|
#define END_WITH_WORKER_LOCK_OPTIONAL(w) while (__cilkrts_worker_unlock(w), 0)
|
|
#endif
|
|
|
|
|
|
#define BEGIN_WITH_FRAME_LOCK(w, ff) \
|
|
do { full_frame *_locked_ff = ff; __cilkrts_frame_lock(w, _locked_ff); do
|
|
|
|
#define END_WITH_FRAME_LOCK(w, ff) \
|
|
while (__cilkrts_frame_unlock(w, _locked_ff), 0); } while (0)
|
|
|
|
/* W becomes the owner of F and F can be stolen from W */
|
|
static void make_runnable(__cilkrts_worker *w, full_frame *ff)
|
|
{
|
|
w->l->frame_ff = ff;
|
|
|
|
/* CALL_STACK is invalid (the information is stored implicitly in W) */
|
|
ff->call_stack = 0;
|
|
}
|
|
|
|
/*
|
|
* The worker parameter is unused, except for print-debugging purposes.
|
|
*/
|
|
static void make_unrunnable(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf,
|
|
int is_loot,
|
|
const char *why)
|
|
{
|
|
/* CALL_STACK becomes valid again */
|
|
ff->call_stack = sf;
|
|
|
|
if (sf) {
|
|
#if CILK_LIB_DEBUG
|
|
if (__builtin_expect(sf->flags & CILK_FRAME_EXITING, 0))
|
|
__cilkrts_bug("W%d suspending exiting frame %p/%p\n", w->self, ff, sf);
|
|
#endif
|
|
sf->flags |= CILK_FRAME_STOLEN | CILK_FRAME_SUSPENDED;
|
|
sf->worker = 0;
|
|
|
|
if (is_loot)
|
|
__cilkrts_put_stack(ff, sf);
|
|
|
|
/* perform any system-dependent action, such as saving the
|
|
state of the stack */
|
|
__cilkrts_make_unrunnable_sysdep(w, ff, sf, is_loot, why);
|
|
}
|
|
}
|
|
|
|
|
|
/* Push the next full frame to be made active in this worker and increment its
|
|
* join counter. __cilkrts_push_next_frame and pop_next_frame work on a
|
|
* one-element queue. This queue is used to communicate across the runtime
|
|
* from the code that wants to activate a frame to the code that can actually
|
|
* begin execution on that frame. They are asymetrical in that push
|
|
* increments the join counter but pop does not decrement it. Rather, a
|
|
* single push/pop combination makes a frame active and increments its join
|
|
* counter once. */
|
|
void __cilkrts_push_next_frame(__cilkrts_worker *w, full_frame *ff)
|
|
{
|
|
CILK_ASSERT(ff);
|
|
CILK_ASSERT(!w->l->next_frame_ff);
|
|
incjoin(ff);
|
|
w->l->next_frame_ff = ff;
|
|
}
|
|
|
|
/* Get the next full-frame to be made active in this worker. The join count
|
|
* of the full frame will have been incremented by the corresponding push
|
|
* event. See __cilkrts_push_next_frame, above.
|
|
*/
|
|
static full_frame *pop_next_frame(__cilkrts_worker *w)
|
|
{
|
|
full_frame *ff;
|
|
ff = w->l->next_frame_ff;
|
|
// Remove the frame from the next_frame field.
|
|
//
|
|
// If this is a user worker, then there is a chance that another worker
|
|
// from our team could push work into our next_frame (if it is the last
|
|
// worker doing work for this team). The other worker's setting of the
|
|
// next_frame could race with our setting of next_frame to NULL. This is
|
|
// the only possible race condition on next_frame. However, if next_frame
|
|
// has a non-NULL value, then it means the team still has work to do, and
|
|
// there is no chance of another team member populating next_frame. Thus,
|
|
// it is safe to set next_frame to NULL, if it was populated. There is no
|
|
// need for an atomic op.
|
|
if (NULL != ff) {
|
|
w->l->next_frame_ff = NULL;
|
|
}
|
|
return ff;
|
|
}
|
|
|
|
/*
|
|
* Identify the single worker that is allowed to cross a sync in this frame. A
|
|
* thief should call this function when it is the first to steal work from a
|
|
* user worker. "First to steal work" may mean that there has been parallelism
|
|
* in the user worker before, but the whole team sync'd, and this is the first
|
|
* steal after that.
|
|
*
|
|
* This should happen while holding the worker and frame lock.
|
|
*/
|
|
static void set_sync_master(__cilkrts_worker *w, full_frame *ff)
|
|
{
|
|
w->l->last_full_frame = ff;
|
|
ff->sync_master = w;
|
|
}
|
|
|
|
/*
|
|
* The sync that ends all parallelism for a particular user worker is about to
|
|
* be crossed. Decouple the worker and frame.
|
|
*
|
|
* No locks need to be held since the user worker isn't doing anything, and none
|
|
* of the system workers can steal from it. But unset_sync_master() should be
|
|
* called before the user worker knows about this work (i.e., before it is
|
|
* inserted into the w->l->next_frame_ff is set).
|
|
*/
|
|
static void unset_sync_master(__cilkrts_worker *w, full_frame *ff)
|
|
{
|
|
CILK_ASSERT(WORKER_USER == w->l->type);
|
|
CILK_ASSERT(ff->sync_master == w);
|
|
ff->sync_master = NULL;
|
|
w->l->last_full_frame = NULL;
|
|
}
|
|
|
|
/********************************************************************
|
|
* THE protocol:
|
|
********************************************************************/
|
|
/*
|
|
* This is a protocol for work stealing that minimizes the overhead on
|
|
* the victim.
|
|
*
|
|
* The protocol uses three shared pointers into the worker's deque:
|
|
* - T - the "tail"
|
|
* - H - the "head"
|
|
* - E - the "exception" NB: In this case, "exception" has nothing to do
|
|
* with C++ throw-catch exceptions -- it refers only to a non-normal return,
|
|
* i.e., a steal or similar scheduling exception.
|
|
*
|
|
* with H <= E, H <= T.
|
|
*
|
|
* Stack frames SF, where H <= E < T, are available for stealing.
|
|
*
|
|
* The worker operates on the T end of the stack. The frame being
|
|
* worked on is not on the stack. To make a continuation available for
|
|
* stealing the worker pushes a from onto the stack: stores *T++ = SF.
|
|
* To return, it pops the frame off the stack: obtains SF = *--T.
|
|
*
|
|
* After decrementing T, the condition E > T signals to the victim that
|
|
* it should invoke the runtime system's "THE" exception handler. The
|
|
* pointer E can become INFINITY, in which case the victim must invoke
|
|
* the THE exception handler as soon as possible.
|
|
*
|
|
* See "The implementation of the Cilk-5 multithreaded language", PLDI 1998,
|
|
* http://portal.acm.org/citation.cfm?doid=277652.277725, for more information
|
|
* on the THE protocol.
|
|
*/
|
|
|
|
/* the infinity value of E */
|
|
#define EXC_INFINITY ((__cilkrts_stack_frame **) (-1))
|
|
|
|
static void increment_E(__cilkrts_worker *victim)
|
|
{
|
|
__cilkrts_stack_frame *volatile *tmp;
|
|
|
|
// The currently executing worker must own the worker lock to touch
|
|
// victim->exc
|
|
ASSERT_WORKER_LOCK_OWNED(victim);
|
|
|
|
tmp = victim->exc;
|
|
if (tmp != EXC_INFINITY) {
|
|
/* On most x86 this pair of operations would be slightly faster
|
|
as an atomic exchange due to the implicit memory barrier in
|
|
an atomic instruction. */
|
|
victim->exc = tmp + 1;
|
|
__cilkrts_fence();
|
|
}
|
|
}
|
|
|
|
static void decrement_E(__cilkrts_worker *victim)
|
|
{
|
|
__cilkrts_stack_frame *volatile *tmp;
|
|
|
|
// The currently executing worker must own the worker lock to touch
|
|
// victim->exc
|
|
ASSERT_WORKER_LOCK_OWNED(victim);
|
|
|
|
tmp = victim->exc;
|
|
if (tmp != EXC_INFINITY) {
|
|
/* On most x86 this pair of operations would be slightly faster
|
|
as an atomic exchange due to the implicit memory barrier in
|
|
an atomic instruction. */
|
|
victim->exc = tmp - 1;
|
|
__cilkrts_fence(); /* memory fence not really necessary */
|
|
}
|
|
}
|
|
|
|
#if 0
|
|
/* for now unused, will be necessary if we implement abort */
|
|
static void signal_THE_exception(__cilkrts_worker *wparent)
|
|
{
|
|
wparent->exc = EXC_INFINITY;
|
|
__cilkrts_fence();
|
|
}
|
|
#endif
|
|
|
|
static void reset_THE_exception(__cilkrts_worker *w)
|
|
{
|
|
// The currently executing worker must own the worker lock to touch
|
|
// w->exc
|
|
ASSERT_WORKER_LOCK_OWNED(w);
|
|
|
|
w->exc = w->head;
|
|
__cilkrts_fence();
|
|
}
|
|
|
|
/* conditions under which victim->head can be stolen: */
|
|
static int can_steal_from(__cilkrts_worker *victim)
|
|
{
|
|
return ((victim->head < victim->tail) &&
|
|
(victim->head < victim->protected_tail));
|
|
}
|
|
|
|
/* Return TRUE if the frame can be stolen, false otherwise */
|
|
static int dekker_protocol(__cilkrts_worker *victim)
|
|
{
|
|
// increment_E and decrement_E are going to touch victim->exc. The
|
|
// currently executing worker must own victim's lock before they can
|
|
// modify it
|
|
ASSERT_WORKER_LOCK_OWNED(victim);
|
|
|
|
/* ASSERT(E >= H); */
|
|
|
|
increment_E(victim);
|
|
|
|
/* ASSERT(E >= H + 1); */
|
|
if (can_steal_from(victim)) {
|
|
/* success, we can steal victim->head and set H <- H + 1
|
|
in detach() */
|
|
return 1;
|
|
} else {
|
|
/* failure, restore previous state */
|
|
decrement_E(victim);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
|
|
/* Link PARENT and CHILD in the spawn tree */
|
|
static full_frame *make_child(__cilkrts_worker *w,
|
|
full_frame *parent_ff,
|
|
__cilkrts_stack_frame *child_sf,
|
|
cilk_fiber *fiber)
|
|
{
|
|
full_frame *child_ff = __cilkrts_make_full_frame(w, child_sf);
|
|
|
|
child_ff->parent = parent_ff;
|
|
push_child(parent_ff, child_ff);
|
|
|
|
//DBGPRINTF("%d- make_child - child_frame: %p, parent_frame: %p, child_sf: %p\n"
|
|
// " parent - parent: %p, left_sibling: %p, right_sibling: %p, rightmost_child: %p\n"
|
|
// " child - parent: %p, left_sibling: %p, right_sibling: %p, rightmost_child: %p\n",
|
|
// w->self, child, parent, child_sf,
|
|
// parent->parent, parent->left_sibling, parent->right_sibling, parent->rightmost_child,
|
|
// child->parent, child->left_sibling, child->right_sibling, child->rightmost_child);
|
|
CILK_ASSERT(parent_ff->call_stack);
|
|
child_ff->is_call_child = (fiber == NULL);
|
|
|
|
/* PLACEHOLDER_FIBER is used as non-null marker indicating that
|
|
child should be treated as a spawn child even though we have not
|
|
yet assigned a real fiber to its parent. */
|
|
if (fiber == PLACEHOLDER_FIBER)
|
|
fiber = NULL; /* Parent actually gets a null fiber, for now */
|
|
|
|
/* perform any system-dependent actions, such as capturing
|
|
parameter passing information */
|
|
/*__cilkrts_make_child_sysdep(child, parent);*/
|
|
|
|
/* Child gets reducer map and stack of parent.
|
|
Parent gets a new map and new stack. */
|
|
child_ff->fiber_self = parent_ff->fiber_self;
|
|
child_ff->sync_master = NULL;
|
|
|
|
if (child_ff->is_call_child) {
|
|
/* Cause segfault on any attempted access. The parent gets
|
|
the child map and stack when the child completes. */
|
|
parent_ff->fiber_self = 0;
|
|
} else {
|
|
parent_ff->fiber_self = fiber;
|
|
}
|
|
|
|
incjoin(parent_ff);
|
|
return child_ff;
|
|
}
|
|
|
|
static inline __cilkrts_stack_frame *__cilkrts_advance_frame(__cilkrts_stack_frame *sf)
|
|
{
|
|
__cilkrts_stack_frame *p = sf->call_parent;
|
|
sf->call_parent = 0;
|
|
return p;
|
|
}
|
|
|
|
/* w should be the currently executing worker.
|
|
* loot_sf is the youngest stack frame in the call stack being
|
|
* unrolled (i.e., the most deeply nested stack frame.)
|
|
*
|
|
* When this method is called for a steal, loot_sf should be on a
|
|
* victim worker which is different from w.
|
|
* For CILK_FORCE_REDUCE, the victim worker will equal w.
|
|
*
|
|
* Before execution, the __cilkrts_stack_frame's have pointers from
|
|
* older to younger, i.e., a __cilkrts_stack_frame points to parent.
|
|
*
|
|
* This method creates a full frame for each __cilkrts_stack_frame in
|
|
* the call stack, with each full frame also pointing to its parent.
|
|
*
|
|
* The method returns the full frame created for loot_sf, i.e., the
|
|
* youngest full frame.
|
|
*/
|
|
static full_frame *unroll_call_stack(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *const loot_sf)
|
|
{
|
|
__cilkrts_stack_frame *sf = loot_sf;
|
|
__cilkrts_stack_frame *rev_sf = 0;
|
|
__cilkrts_stack_frame *t_sf;
|
|
|
|
CILK_ASSERT(sf);
|
|
/*CILK_ASSERT(sf->call_parent != sf);*/
|
|
|
|
/* The leafmost frame is unsynched. */
|
|
if (sf->worker != w)
|
|
sf->flags |= CILK_FRAME_UNSYNCHED;
|
|
|
|
/* Reverse the call stack to make a linked list ordered from parent
|
|
to child. sf->call_parent points to the child of SF instead of
|
|
the parent. */
|
|
do {
|
|
t_sf = (sf->flags & (CILK_FRAME_DETACHED|CILK_FRAME_STOLEN|CILK_FRAME_LAST))? 0 : sf->call_parent;
|
|
sf->call_parent = rev_sf;
|
|
rev_sf = sf;
|
|
sf = t_sf;
|
|
} while (sf);
|
|
sf = rev_sf;
|
|
|
|
/* Promote each stack frame to a full frame in order from parent
|
|
to child, following the reversed list we just built. */
|
|
make_unrunnable(w, ff, sf, sf == loot_sf, "steal 1");
|
|
/* T is the *child* of SF, because we have reversed the list */
|
|
for (t_sf = __cilkrts_advance_frame(sf); t_sf;
|
|
sf = t_sf, t_sf = __cilkrts_advance_frame(sf)) {
|
|
ff = make_child(w, ff, t_sf, NULL);
|
|
make_unrunnable(w, ff, t_sf, t_sf == loot_sf, "steal 2");
|
|
}
|
|
|
|
/* XXX What if the leafmost frame does not contain a sync
|
|
and this steal is from promote own deque? */
|
|
/*sf->flags |= CILK_FRAME_UNSYNCHED;*/
|
|
|
|
CILK_ASSERT(!sf->call_parent);
|
|
return ff;
|
|
}
|
|
|
|
/* detach the top of the deque frame from the VICTIM and install a new
|
|
CHILD frame in its place */
|
|
static void detach_for_steal(__cilkrts_worker *w,
|
|
__cilkrts_worker *victim,
|
|
cilk_fiber* fiber)
|
|
{
|
|
/* ASSERT: we own victim->lock */
|
|
|
|
full_frame *parent_ff, *child_ff, *loot_ff;
|
|
__cilkrts_stack_frame *volatile *h;
|
|
__cilkrts_stack_frame *sf;
|
|
|
|
w->l->team = victim->l->team;
|
|
|
|
CILK_ASSERT(w->l->frame_ff == 0 || w == victim);
|
|
|
|
h = victim->head;
|
|
|
|
CILK_ASSERT(*h);
|
|
|
|
victim->head = h + 1;
|
|
|
|
parent_ff = victim->l->frame_ff;
|
|
BEGIN_WITH_FRAME_LOCK(w, parent_ff) {
|
|
/* parent no longer referenced by victim */
|
|
decjoin(parent_ff);
|
|
|
|
/* obtain the victim call stack */
|
|
sf = *h;
|
|
|
|
/* perform system-dependent normalizations */
|
|
/*__cilkrts_normalize_call_stack_on_steal(sf);*/
|
|
|
|
/* unroll PARENT_FF with call stack SF, adopt the youngest
|
|
frame LOOT. If loot_ff == parent_ff, then we hold loot_ff->lock,
|
|
otherwise, loot_ff is newly created and we can modify it without
|
|
holding its lock. */
|
|
loot_ff = unroll_call_stack(w, parent_ff, sf);
|
|
|
|
#if REDPAR_DEBUG >= 3
|
|
fprintf(stderr, "[W=%d, victim=%d, desc=detach, parent_ff=%p, loot=%p]\n",
|
|
w->self, victim->self,
|
|
parent_ff, loot_ff);
|
|
#endif
|
|
|
|
if (WORKER_USER == victim->l->type &&
|
|
NULL == victim->l->last_full_frame) {
|
|
// Mark this looted frame as special: only the original user worker
|
|
// may cross the sync.
|
|
//
|
|
// This call is a shared access to
|
|
// victim->l->last_full_frame.
|
|
set_sync_master(victim, loot_ff);
|
|
}
|
|
|
|
/* LOOT is the next frame that the thief W is supposed to
|
|
run, unless the thief is stealing from itself, in which
|
|
case the thief W == VICTIM executes CHILD and nobody
|
|
executes LOOT. */
|
|
if (w == victim) {
|
|
/* Pretend that frame has been stolen */
|
|
loot_ff->call_stack->flags |= CILK_FRAME_UNSYNCHED;
|
|
loot_ff->simulated_stolen = 1;
|
|
}
|
|
else
|
|
__cilkrts_push_next_frame(w, loot_ff);
|
|
|
|
// After this "push_next_frame" call, w now owns loot_ff.
|
|
child_ff = make_child(w, loot_ff, 0, fiber);
|
|
|
|
BEGIN_WITH_FRAME_LOCK(w, child_ff) {
|
|
/* install child in the victim's work queue, taking
|
|
the parent_ff's place */
|
|
/* child is referenced by victim */
|
|
incjoin(child_ff);
|
|
|
|
// With this call, w is bestowing ownership of the newly
|
|
// created frame child_ff to the victim, and victim is
|
|
// giving up ownership of parent_ff.
|
|
//
|
|
// Worker w will either take ownership of parent_ff
|
|
// if parent_ff == loot_ff, or parent_ff will be
|
|
// suspended.
|
|
//
|
|
// Note that this call changes the victim->frame_ff
|
|
// while the victim may be executing.
|
|
make_runnable(victim, child_ff);
|
|
} END_WITH_FRAME_LOCK(w, child_ff);
|
|
} END_WITH_FRAME_LOCK(w, parent_ff);
|
|
}
|
|
|
|
/**
|
|
* @brief cilk_fiber_proc that resumes user code after a successful
|
|
* random steal.
|
|
|
|
* This function longjmps back into the user code whose state is
|
|
* stored in cilk_fiber_get_data(fiber)->resume_sf. The stack pointer
|
|
* is adjusted so that the code resumes on the specified fiber stack
|
|
* instead of its original stack.
|
|
*
|
|
* This method gets executed only on a fiber freshly allocated from a
|
|
* pool.
|
|
*
|
|
* @param fiber The fiber being used to resume user code.
|
|
* @param arg Unused.
|
|
*/
|
|
static
|
|
void fiber_proc_to_resume_user_code_for_random_steal(cilk_fiber *fiber)
|
|
{
|
|
cilk_fiber_data *data = cilk_fiber_get_data(fiber);
|
|
__cilkrts_stack_frame* sf = data->resume_sf;
|
|
full_frame *ff;
|
|
|
|
CILK_ASSERT(sf);
|
|
|
|
// When we pull the resume_sf out of the fiber to resume it, clear
|
|
// the old value.
|
|
data->resume_sf = NULL;
|
|
CILK_ASSERT(sf->worker == data->owner);
|
|
ff = sf->worker->l->frame_ff;
|
|
|
|
// For Win32, we need to overwrite the default exception handler
|
|
// in this function, so that when the OS exception handling code
|
|
// walks off the top of the current Cilk stack, it reaches our stub
|
|
// handler.
|
|
|
|
// Also, this function needs to be wrapped into a try-catch block
|
|
// so the compiler generates the appropriate exception information
|
|
// in this frame.
|
|
|
|
// TBD: IS THIS HANDLER IN THE WRONG PLACE? Can we longjmp out of
|
|
// this function (and does it matter?)
|
|
#if defined(_WIN32) && !defined(_WIN64)
|
|
install_exception_stub_handler();
|
|
__try
|
|
#endif
|
|
{
|
|
char* new_sp = sysdep_reset_jump_buffers_for_resume(fiber, ff, sf);
|
|
|
|
// Notify the Intel tools that we're stealing code
|
|
ITT_SYNC_ACQUIRED(sf->worker);
|
|
NOTIFY_ZC_INTRINSIC("cilk_continue", sf);
|
|
|
|
// TBD: We'd like to move TBB-interop methods into the fiber
|
|
// eventually.
|
|
cilk_fiber_invoke_tbb_stack_op(fiber, CILK_TBB_STACK_ADOPT);
|
|
|
|
sf->flags &= ~CILK_FRAME_SUSPENDED;
|
|
|
|
// longjmp to user code. Don't process exceptions here,
|
|
// because we are resuming a stolen frame.
|
|
sysdep_longjmp_to_sf(new_sp, sf, NULL);
|
|
/*NOTREACHED*/
|
|
// Intel's C compiler respects the preceding lint pragma
|
|
}
|
|
#if defined(_WIN32) && !defined(_WIN64)
|
|
__except (CILK_ASSERT(!"should not execute the the stub filter"),
|
|
EXCEPTION_EXECUTE_HANDLER)
|
|
{
|
|
// If we are here, that means something very wrong
|
|
// has happened in our exception processing...
|
|
CILK_ASSERT(! "should not be here!");
|
|
}
|
|
#endif
|
|
}
|
|
|
|
static void random_steal(__cilkrts_worker *w)
|
|
{
|
|
__cilkrts_worker *victim = NULL;
|
|
cilk_fiber *fiber = NULL;
|
|
int n;
|
|
int success = 0;
|
|
int32_t victim_id;
|
|
|
|
// Nothing's been stolen yet. When true, this will flag
|
|
// setup_for_execution_pedigree to increment the pedigree
|
|
w->l->work_stolen = 0;
|
|
|
|
/* If the user has disabled stealing (using the debugger) we fail */
|
|
if (__builtin_expect(w->g->stealing_disabled, 0))
|
|
return;
|
|
|
|
CILK_ASSERT(w->l->type == WORKER_SYSTEM || w->l->team == w);
|
|
|
|
/* If there is only one processor work can still be stolen.
|
|
There must be only one worker to prevent stealing. */
|
|
CILK_ASSERT(w->g->total_workers > 1);
|
|
|
|
/* pick random *other* victim */
|
|
n = myrand(w) % (w->g->total_workers - 1);
|
|
if (n >= w->self)
|
|
++n;
|
|
|
|
// If we're replaying a log, override the victim. -1 indicates that
|
|
// we've exhausted the list of things this worker stole when we recorded
|
|
// the log so just return. If we're not replaying a log,
|
|
// replay_get_next_recorded_victim() just returns the victim ID passed in.
|
|
n = replay_get_next_recorded_victim(w, n);
|
|
if (-1 == n)
|
|
return;
|
|
|
|
victim = w->g->workers[n];
|
|
|
|
START_INTERVAL(w, INTERVAL_FIBER_ALLOCATE) {
|
|
/* Verify that we can get a stack. If not, no need to continue. */
|
|
fiber = cilk_fiber_allocate(&w->l->fiber_pool);
|
|
} STOP_INTERVAL(w, INTERVAL_FIBER_ALLOCATE);
|
|
|
|
|
|
if (NULL == fiber) {
|
|
#if FIBER_DEBUG >= 2
|
|
fprintf(stderr, "w=%d: failed steal because we could not get a fiber\n",
|
|
w->self);
|
|
#endif
|
|
return;
|
|
}
|
|
|
|
/* do not steal from self */
|
|
CILK_ASSERT (victim != w);
|
|
|
|
/* Execute a quick check before engaging in the THE protocol.
|
|
Avoid grabbing locks if there is nothing to steal. */
|
|
if (!can_steal_from(victim)) {
|
|
NOTE_INTERVAL(w, INTERVAL_STEAL_FAIL_EMPTYQ);
|
|
START_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE) {
|
|
int ref_count = cilk_fiber_remove_reference(fiber, &w->l->fiber_pool);
|
|
// Fibers we use when trying to steal should not be active,
|
|
// and thus should not have any other references.
|
|
CILK_ASSERT(0 == ref_count);
|
|
} STOP_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE);
|
|
return;
|
|
}
|
|
|
|
/* Attempt to steal work from the victim */
|
|
if (worker_trylock_other(w, victim)) {
|
|
if (w->l->type == WORKER_USER && victim->l->team != w) {
|
|
|
|
// Fail to steal if this is a user worker and the victim is not
|
|
// on this team. If a user worker were allowed to steal work
|
|
// descended from another user worker, the former might not be
|
|
// done with its work by the time it was needed to resume and
|
|
// unbind. Therefore, user workers are not permitted to change
|
|
// teams.
|
|
|
|
// There is no race on the victim's team because the victim cannot
|
|
// change its team until it runs out of work to do, at which point
|
|
// it will try to take out its own lock, and this worker already
|
|
// holds it.
|
|
NOTE_INTERVAL(w, INTERVAL_STEAL_FAIL_USER_WORKER);
|
|
|
|
} else if (victim->l->frame_ff) {
|
|
// A successful steal will change victim->frame_ff, even
|
|
// though the victim may be executing. Thus, the lock on
|
|
// the victim's deque is also protecting victim->frame_ff.
|
|
if (dekker_protocol(victim)) {
|
|
int proceed_with_steal = 1; // optimistic
|
|
|
|
// If we're replaying a log, verify that this the correct frame
|
|
// to steal from the victim
|
|
if (! replay_match_victim_pedigree(w, victim))
|
|
{
|
|
// Abort the steal attempt. decrement_E(victim) to
|
|
// counter the increment_E(victim) done by the
|
|
// dekker protocol
|
|
decrement_E(victim);
|
|
proceed_with_steal = 0;
|
|
}
|
|
|
|
if (proceed_with_steal)
|
|
{
|
|
START_INTERVAL(w, INTERVAL_STEAL_SUCCESS) {
|
|
success = 1;
|
|
detach_for_steal(w, victim, fiber);
|
|
victim_id = victim->self;
|
|
|
|
#if REDPAR_DEBUG >= 1
|
|
fprintf(stderr, "Wkr %d stole from victim %d, fiber = %p\n",
|
|
w->self, victim->self, fiber);
|
|
#endif
|
|
|
|
// The use of victim->self contradicts our
|
|
// classification of the "self" field as
|
|
// local. But since this code is only for
|
|
// debugging, it is ok.
|
|
DBGPRINTF ("%d-%p: Stealing work from worker %d\n"
|
|
" sf: %p, call parent: %p\n",
|
|
w->self, GetCurrentFiber(), victim->self,
|
|
w->l->next_frame_ff->call_stack,
|
|
w->l->next_frame_ff->call_stack->call_parent);
|
|
} STOP_INTERVAL(w, INTERVAL_STEAL_SUCCESS);
|
|
} // end if(proceed_with_steal)
|
|
} else {
|
|
NOTE_INTERVAL(w, INTERVAL_STEAL_FAIL_DEKKER);
|
|
}
|
|
} else {
|
|
NOTE_INTERVAL(w, INTERVAL_STEAL_FAIL_EMPTYQ);
|
|
}
|
|
worker_unlock_other(w, victim);
|
|
} else {
|
|
NOTE_INTERVAL(w, INTERVAL_STEAL_FAIL_LOCK);
|
|
}
|
|
|
|
// Record whether work was stolen. When true, this will flag
|
|
// setup_for_execution_pedigree to increment the pedigree
|
|
w->l->work_stolen = success;
|
|
|
|
if (0 == success) {
|
|
// failed to steal work. Return the fiber to the pool.
|
|
START_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE) {
|
|
int ref_count = cilk_fiber_remove_reference(fiber, &w->l->fiber_pool);
|
|
// Fibers we use when trying to steal should not be active,
|
|
// and thus should not have any other references.
|
|
CILK_ASSERT(0 == ref_count);
|
|
} STOP_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE);
|
|
}
|
|
else
|
|
{
|
|
// Since our steal was successful, finish initialization of
|
|
// the fiber.
|
|
cilk_fiber_reset_state(fiber,
|
|
fiber_proc_to_resume_user_code_for_random_steal);
|
|
// Record the pedigree of the frame that w has stolen.
|
|
// record only if CILK_RECORD_LOG is set
|
|
replay_record_steal(w, victim_id);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* At a provably good steal, we need to transfer the child reducer map
|
|
* from ff->children_reducer_map into v->reducer_map, where v is the
|
|
* worker that resumes execution of ff.
|
|
*
|
|
* Normally, we have v == w, where w is the currently executing
|
|
* worker. In the case where we are resuming a team leader on a user
|
|
* worker, however, v might differ from w.
|
|
|
|
* Thus, this, operation is a no-op, since we can't really move
|
|
* ff->children_reducer_map into w here.
|
|
*
|
|
* Instead, this work is done in setup_for_execution_reducers().
|
|
*/
|
|
static inline void provably_good_steal_reducers(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// No-op.
|
|
}
|
|
|
|
/* at a provably good steal, incorporate the accumulated exceptions of
|
|
children into the parent's exception */
|
|
static void provably_good_steal_exceptions(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we own ff->lock
|
|
ff->pending_exception =
|
|
__cilkrts_merge_pending_exceptions(w,
|
|
ff->child_pending_exception,
|
|
ff->pending_exception);
|
|
ff->child_pending_exception = NULL;
|
|
}
|
|
|
|
/* At sync discard the frame's old stack and take the leftmost child's. */
|
|
static void provably_good_steal_stacks(__cilkrts_worker *w, full_frame *ff)
|
|
{
|
|
CILK_ASSERT(NULL == ff->fiber_self);
|
|
ff->fiber_self = ff->fiber_child;
|
|
ff->fiber_child = NULL;
|
|
}
|
|
|
|
static void __cilkrts_mark_synched(full_frame *ff)
|
|
{
|
|
ff->call_stack->flags &= ~CILK_FRAME_UNSYNCHED;
|
|
ff->simulated_stolen = 0;
|
|
}
|
|
|
|
static
|
|
enum provably_good_steal_t provably_good_steal(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we hold w->lock and ff->lock
|
|
|
|
enum provably_good_steal_t result = ABANDON_EXECUTION;
|
|
|
|
// If the current replay entry is a sync record matching the worker's
|
|
// pedigree, AND this isn't the last child to the sync, return
|
|
// WAIT_FOR_CONTINUE to indicate that the caller should loop until
|
|
// we find the right frame to steal and CONTINUE_EXECUTION is returned.
|
|
int match_found = replay_match_sync_pedigree(w);
|
|
if (match_found && (0 != simulate_decjoin(ff)))
|
|
return WAIT_FOR_CONTINUE;
|
|
|
|
START_INTERVAL(w, INTERVAL_PROVABLY_GOOD_STEAL) {
|
|
if (decjoin(ff) == 0) {
|
|
provably_good_steal_reducers(w, ff);
|
|
provably_good_steal_exceptions(w, ff);
|
|
provably_good_steal_stacks(w, ff);
|
|
__cilkrts_mark_synched(ff);
|
|
|
|
// If the original owner wants this frame back (to resume
|
|
// it on its original thread) pass it back now.
|
|
if (NULL != ff->sync_master) {
|
|
// The frame wants to go back and be executed by the original
|
|
// user thread. We can throw caution to the wind and push the
|
|
// frame straight onto its queue because the only way we have
|
|
// gotten to this point of being able to continue execution of
|
|
// the frame is if the original user worker is spinning without
|
|
// work.
|
|
|
|
unset_sync_master(w->l->team, ff);
|
|
__cilkrts_push_next_frame(w->l->team, ff);
|
|
|
|
// If this is the team leader we're not abandoning the work
|
|
if (w == w->l->team)
|
|
result = CONTINUE_EXECUTION;
|
|
} else {
|
|
__cilkrts_push_next_frame(w, ff);
|
|
result = CONTINUE_EXECUTION; // Continue working on this thread
|
|
}
|
|
|
|
// The __cilkrts_push_next_frame() call changes ownership
|
|
// of ff to the specified worker.
|
|
}
|
|
} STOP_INTERVAL(w, INTERVAL_PROVABLY_GOOD_STEAL);
|
|
|
|
// Only write a SYNC record if:
|
|
// - We're recording a log *AND*
|
|
// - We're the worker continuing from this sync
|
|
replay_record_sync(w, result == CONTINUE_EXECUTION);
|
|
|
|
// If we're replaying a log, and matched a sync from the log, mark the
|
|
// sync record seen if the sync isn't going to be abandoned.
|
|
replay_advance_from_sync (w, match_found, result == CONTINUE_EXECUTION);
|
|
|
|
return result;
|
|
}
|
|
|
|
static void unconditional_steal(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we hold ff->lock
|
|
|
|
START_INTERVAL(w, INTERVAL_UNCONDITIONAL_STEAL) {
|
|
decjoin(ff);
|
|
__cilkrts_push_next_frame(w, ff);
|
|
} STOP_INTERVAL(w, INTERVAL_UNCONDITIONAL_STEAL);
|
|
}
|
|
|
|
|
|
/* CHILD is about to die. Give its exceptions to a sibling or to the
|
|
parent. */
|
|
static inline void splice_exceptions_for_call(__cilkrts_worker *w,
|
|
full_frame *parent_ff,
|
|
full_frame *child_ff)
|
|
{
|
|
// ASSERT: We own parent_ff->lock
|
|
CILK_ASSERT(child_ff->is_call_child);
|
|
CILK_ASSERT(NULL == child_ff->right_pending_exception);
|
|
CILK_ASSERT(NULL == parent_ff->pending_exception);
|
|
|
|
parent_ff->pending_exception = child_ff->pending_exception;
|
|
child_ff->pending_exception = NULL;
|
|
}
|
|
|
|
/**
|
|
* Merge exceptions for a dying child.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The child frame that is dying.
|
|
* @param left_exception_ptr Pointer to the exception that is to our left.
|
|
*/
|
|
static inline
|
|
void splice_exceptions_for_spawn(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
struct pending_exception_info **left_exception_ptr)
|
|
{
|
|
// ASSERT: parent_ff == child_ff->parent.
|
|
// ASSERT: We own parent_ff->lock
|
|
|
|
// Merge current exception into the slot where the left
|
|
// exception should go.
|
|
*left_exception_ptr =
|
|
__cilkrts_merge_pending_exceptions(w,
|
|
*left_exception_ptr,
|
|
ff->pending_exception);
|
|
ff->pending_exception = NULL;
|
|
|
|
|
|
// Merge right exception into the slot where the left exception
|
|
// should go.
|
|
*left_exception_ptr =
|
|
__cilkrts_merge_pending_exceptions(w,
|
|
*left_exception_ptr,
|
|
ff->right_pending_exception);
|
|
ff->right_pending_exception = NULL;
|
|
}
|
|
|
|
|
|
static inline void splice_stacks_for_call(__cilkrts_worker *w,
|
|
full_frame *parent_ff,
|
|
full_frame *child_ff)
|
|
{
|
|
#if CILK_LIB_DEBUG
|
|
if (parent_ff->call_stack)
|
|
CILK_ASSERT(!(parent_ff->call_stack->flags & CILK_FRAME_MBZ));
|
|
#endif
|
|
|
|
/* A synched frame does not have accumulated child reducers. */
|
|
CILK_ASSERT(!child_ff->fiber_child);
|
|
CILK_ASSERT(child_ff->is_call_child);
|
|
|
|
/* An attached parent has no self fiber. It may have
|
|
accumulated child fibers or child owners, which should be
|
|
ignored until sync. */
|
|
CILK_ASSERT(!parent_ff->fiber_self);
|
|
parent_ff->fiber_self = child_ff->fiber_self;
|
|
child_ff->fiber_self = NULL;
|
|
}
|
|
|
|
static void finalize_child_for_call(__cilkrts_worker *w,
|
|
full_frame *parent_ff,
|
|
full_frame *child_ff)
|
|
{
|
|
// ASSERT: we hold w->lock and parent_ff->lock
|
|
|
|
START_INTERVAL(w, INTERVAL_FINALIZE_CHILD) {
|
|
CILK_ASSERT(child_ff->is_call_child);
|
|
CILK_ASSERT(child_ff->join_counter == 0);
|
|
CILK_ASSERT(!child_ff->rightmost_child);
|
|
CILK_ASSERT(child_ff == parent_ff->rightmost_child);
|
|
|
|
// CHILD is about to die.
|
|
// Splicing out reducers is a no-op for a call since
|
|
// w->reducer_map should already store the correct
|
|
// reducer map.
|
|
|
|
// ASSERT there are no maps left to reduce.
|
|
CILK_ASSERT(NULL == child_ff->children_reducer_map);
|
|
CILK_ASSERT(NULL == child_ff->right_reducer_map);
|
|
|
|
splice_exceptions_for_call(w, parent_ff, child_ff);
|
|
|
|
splice_stacks_for_call(w, parent_ff, child_ff);
|
|
|
|
/* remove CHILD from list of children of PARENT */
|
|
unlink_child(parent_ff, child_ff);
|
|
|
|
/* continue with the parent. */
|
|
unconditional_steal(w, parent_ff);
|
|
__cilkrts_destroy_full_frame(w, child_ff);
|
|
} STOP_INTERVAL(w, INTERVAL_FINALIZE_CHILD);
|
|
}
|
|
|
|
|
|
/**
|
|
* The invariant on ff->children_reducer_map is that when ff is
|
|
* synched and when we are about to resume execution of ff, at least
|
|
* one of ff->children_reducer_map and w->reducer_map must be NULL.
|
|
*
|
|
* Consider the two possibilities before resuming execution of ff:
|
|
*
|
|
* 1. Suppose ff is synched and suspended. Then either
|
|
*
|
|
* (a) ff->children_reducer_map stores the reducer map that w
|
|
* should use, where w is the worker resuming execution of ff,
|
|
* OR
|
|
* (b) w already has a user map, and ff->children_reducer_map is NULL.
|
|
*
|
|
* Case (a) happens when we are resuming execution of ff as a
|
|
* provably good steal. In this case, w->reducer_map should be
|
|
* NULL and ff->children_reducer_map is valid. To resume
|
|
* execution of ff on w, set w->reducer_map to
|
|
* ff->children_reducer_map.
|
|
*
|
|
* Case (b) occurs when we resume execution of ff because ff is a
|
|
* called child. Then, ff->children_reducer_map should be NULL,
|
|
* and w should already have a valid reducer map when resuming
|
|
* execution of ff. We resume execution of ff without changing
|
|
* w->reducer_map.
|
|
*
|
|
* 2. Suppose frame ff is not synched (i.e., it is active and might have
|
|
* active children). Then ff->children_reducer_map is the slot for
|
|
* storing the reducer map from ff's leftmost child, as in the reducer
|
|
* protocol. The runtime may resume execution of ff while it is not
|
|
* synched only because of a steal.
|
|
* In this case, while we are resuming ff, ff->children_reducer_map
|
|
* may be non-NULL (because one of ff's children has completed).
|
|
* We resume execution of ff without changing w->reducer_map.
|
|
*/
|
|
static void setup_for_execution_reducers(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// We only need to move ff->children_reducer_map into
|
|
// w->reducer_map in case 1(a).
|
|
//
|
|
// First check whether ff is synched.
|
|
__cilkrts_stack_frame *sf = ff->call_stack;
|
|
if (!(sf->flags & CILK_FRAME_UNSYNCHED)) {
|
|
// In this case, ff is synched. (Case 1).
|
|
CILK_ASSERT(!ff->rightmost_child);
|
|
|
|
// Test whether we are in case 1(a) and have
|
|
// something to do. Note that if both
|
|
// ff->children_reducer_map and w->reducer_map are NULL, we
|
|
// can't distinguish between cases 1(a) and 1(b) here.
|
|
if (ff->children_reducer_map) {
|
|
// We are in Case 1(a).
|
|
CILK_ASSERT(!w->reducer_map);
|
|
w->reducer_map = ff->children_reducer_map;
|
|
ff->children_reducer_map = NULL;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void setup_for_execution_exceptions(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
CILK_ASSERT(NULL == w->l->pending_exception);
|
|
w->l->pending_exception = ff->pending_exception;
|
|
ff->pending_exception = NULL;
|
|
}
|
|
|
|
#if 0 /* unused */
|
|
static void setup_for_execution_stack(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* setup_for_execution_pedigree
|
|
*
|
|
* Copies the pedigree information from the frame we're resuming to the
|
|
* worker. Increments the pedigree if this is work that has been stolen
|
|
* to match the increment on a return from a spawn helper.
|
|
*/
|
|
static void setup_for_execution_pedigree(__cilkrts_worker *w)
|
|
{
|
|
int pedigree_unsynched;
|
|
__cilkrts_stack_frame *sf = w->current_stack_frame;
|
|
|
|
CILK_ASSERT(NULL != sf);
|
|
|
|
// If this isn't an ABI 1 or later frame, there's no pedigree information
|
|
if (0 == CILK_FRAME_VERSION_VALUE(sf->flags))
|
|
return;
|
|
|
|
// Note whether the pedigree is unsynched and clear the flag before
|
|
// we forget
|
|
pedigree_unsynched = sf->flags & CILK_FRAME_SF_PEDIGREE_UNSYNCHED;
|
|
sf->flags &= ~CILK_FRAME_SF_PEDIGREE_UNSYNCHED;
|
|
|
|
// If we're just marshalling onto this worker, do not increment
|
|
// the rank since that wouldn't happen in a sequential execution
|
|
if (w->l->work_stolen || pedigree_unsynched)
|
|
{
|
|
if (w->l->work_stolen)
|
|
w->pedigree.rank = sf->parent_pedigree.rank + 1;
|
|
else
|
|
w->pedigree.rank = sf->parent_pedigree.rank;
|
|
}
|
|
|
|
w->pedigree.parent = sf->parent_pedigree.parent;
|
|
w->l->work_stolen = 0;
|
|
}
|
|
|
|
static void setup_for_execution(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
int is_return_from_call)
|
|
{
|
|
// ASSERT: We own w->lock and ff->lock || P == 1
|
|
|
|
setup_for_execution_reducers(w, ff);
|
|
setup_for_execution_exceptions(w, ff);
|
|
/*setup_for_execution_stack(w, ff);*/
|
|
|
|
ff->call_stack->worker = w;
|
|
w->current_stack_frame = ff->call_stack;
|
|
|
|
// If this is a return from a call, leave the pedigree alone
|
|
if (! is_return_from_call)
|
|
setup_for_execution_pedigree(w);
|
|
|
|
__cilkrts_setup_for_execution_sysdep(w, ff);
|
|
|
|
w->head = w->tail = w->l->ltq;
|
|
reset_THE_exception(w);
|
|
|
|
make_runnable(w, ff);
|
|
}
|
|
|
|
|
|
/*
|
|
* Called by the scheduling fiber, right before
|
|
* resuming a sf/ff for user code.
|
|
*
|
|
* This method associates the specified sf with the worker.
|
|
*
|
|
* It also asserts that w, ff, and sf all have the expected properties
|
|
* for resuming user code.
|
|
*/
|
|
void scheduling_fiber_prepare_to_resume_user_code(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf)
|
|
{
|
|
w->current_stack_frame = sf;
|
|
sf->worker = w;
|
|
|
|
// Lots of debugging checks on the state of the fiber we might be
|
|
// resuming.
|
|
#if FIBER_DEBUG >= 1
|
|
# if FIBER_DEBUG >= 3
|
|
{
|
|
fprintf(stderr, "w=%d: ff=%p, sf=%p. about to resume user code\n",
|
|
w->self, ff, sf);
|
|
}
|
|
# endif
|
|
|
|
const int flags = sf->flags;
|
|
CILK_ASSERT(flags & CILK_FRAME_SUSPENDED);
|
|
CILK_ASSERT(!sf->call_parent);
|
|
CILK_ASSERT(w->head == w->tail);
|
|
|
|
/* A frame can not be resumed unless it was suspended. */
|
|
CILK_ASSERT(ff->sync_sp != NULL);
|
|
|
|
/* The leftmost frame has no allocated stack */
|
|
if (ff->simulated_stolen)
|
|
CILK_ASSERT(flags & CILK_FRAME_UNSYNCHED);
|
|
else if (flags & CILK_FRAME_UNSYNCHED)
|
|
/* XXX By coincidence sync_sp could be null. */
|
|
CILK_ASSERT(ff->fiber_self != NULL);
|
|
else
|
|
/* XXX This frame could be resumed unsynched on the leftmost stack */
|
|
CILK_ASSERT((ff->sync_master == 0 || ff->sync_master == w));
|
|
CILK_ASSERT(w->l->frame_ff == ff);
|
|
#endif
|
|
}
|
|
|
|
|
|
/**
|
|
* This method is the first method that should execute after we've
|
|
* switched to a scheduling fiber from user code.
|
|
*
|
|
* @param fiber The scheduling fiber for the current worker.
|
|
* @param wptr The current worker.
|
|
*/
|
|
static void enter_runtime_transition_proc(cilk_fiber *fiber)
|
|
{
|
|
// We can execute this method for one of three reasons:
|
|
// 1. Undo-detach finds parent stolen.
|
|
// 2. Sync suspends frame.
|
|
// 3. Return from Cilk entry point.
|
|
//
|
|
//
|
|
// In cases 1 and 2, the frame may be truly suspended or
|
|
// may be immediately executed by this worker after provably_good_steal.
|
|
//
|
|
//
|
|
// There is a fourth case, which can, but does not need to execute
|
|
// this function:
|
|
// 4. Starting up the scheduling loop on a user or
|
|
// system worker. In this case, we won't have
|
|
// a scheduling stack function to run.
|
|
__cilkrts_worker* w = cilk_fiber_get_owner(fiber);
|
|
if (w->l->post_suspend) {
|
|
// Run the continuation function passed to longjmp_into_runtime
|
|
run_scheduling_stack_fcn(w);
|
|
|
|
// After we have jumped into the runtime and run the
|
|
// scheduling function, any reducer map the worker had before entering the runtime
|
|
// should have already been saved into the appropriate full
|
|
// frame.
|
|
CILK_ASSERT(NULL == w->reducer_map);
|
|
|
|
// There shouldn't be any uncaught exceptions.
|
|
//
|
|
// In Windows, the OS catches any exceptions not caught by the
|
|
// user code. Thus, we are omitting the check on Windows.
|
|
//
|
|
// On Android, calling std::uncaught_exception with the stlport
|
|
// library causes a seg fault. Since we're not supporting
|
|
// exceptions there at this point, just don't do the check
|
|
//
|
|
// TBD: Is this check also safe to do on Windows?
|
|
CILKBUG_ASSERT_NO_UNCAUGHT_EXCEPTION();
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Method called to jump back to executing user code.
|
|
*
|
|
* A normal return from the runtime back to resuming user code calls
|
|
* this method. A computation executed using force_reduce also calls
|
|
* this method to return to user code.
|
|
*
|
|
* This function should not contain any code that depends on a fiber.
|
|
* In a force-reduce case, the user worker may not have a fiber. In
|
|
* the force-reduce case, we call this method directly instead of
|
|
* calling @c user_code_resume_after_switch_into_runtime.
|
|
*/
|
|
static inline NORETURN
|
|
cilkrts_resume(__cilkrts_stack_frame *sf, full_frame *ff)
|
|
{
|
|
// Save the sync stack pointer, and do the bookkeeping
|
|
char* sync_sp = ff->sync_sp;
|
|
__cilkrts_take_stack(ff, sync_sp); // leaves ff->sync_sp null
|
|
|
|
sf->flags &= ~CILK_FRAME_SUSPENDED;
|
|
// Actually longjmp to the user code.
|
|
// We may have exceptions to deal with, since we are resuming
|
|
// a previous-suspended frame.
|
|
sysdep_longjmp_to_sf(sync_sp, sf, ff);
|
|
}
|
|
|
|
|
|
/**
|
|
* Called by the user-code fiber right before resuming a full frame
|
|
* (sf/ff).
|
|
*
|
|
* This method pulls sf/ff out of the worker, and then calls
|
|
* cilkrts_resume to jump to user code.
|
|
*/
|
|
static NORETURN
|
|
user_code_resume_after_switch_into_runtime(cilk_fiber *fiber)
|
|
{
|
|
__cilkrts_worker *w = cilk_fiber_get_owner(fiber);
|
|
__cilkrts_stack_frame *sf;
|
|
full_frame *ff;
|
|
sf = w->current_stack_frame;
|
|
ff = sf->worker->l->frame_ff;
|
|
|
|
#if FIBER_DEBUG >= 1
|
|
CILK_ASSERT(ff->fiber_self == fiber);
|
|
cilk_fiber_data *fdata = cilk_fiber_get_data(fiber);
|
|
DBGPRINTF ("%d-%p: resume_after_switch_into_runtime, fiber=%p\n",
|
|
w->self, w, fiber);
|
|
CILK_ASSERT(sf == fdata->resume_sf);
|
|
#endif
|
|
|
|
// Notify the Intel tools that we're stealing code
|
|
ITT_SYNC_ACQUIRED(sf->worker);
|
|
NOTIFY_ZC_INTRINSIC("cilk_continue", sf);
|
|
cilk_fiber_invoke_tbb_stack_op(fiber, CILK_TBB_STACK_ADOPT);
|
|
|
|
// Actually jump to user code.
|
|
cilkrts_resume(sf, ff);
|
|
}
|
|
|
|
|
|
/* The current stack is about to either be suspended or destroyed. This
|
|
* function will switch to the stack on which the scheduler is suspended and
|
|
* resume running the scheduler within function do_work(). Upon waking up,
|
|
* the scheduler will run the 'cont' function, using the supplied worker and
|
|
* frame.
|
|
*/
|
|
static NORETURN
|
|
longjmp_into_runtime(__cilkrts_worker *w,
|
|
scheduling_stack_fcn_t fcn,
|
|
__cilkrts_stack_frame *sf)
|
|
{
|
|
full_frame *ff, *ff2;
|
|
|
|
CILK_ASSERT(!w->l->post_suspend);
|
|
ff = w->l->frame_ff;
|
|
|
|
// If we've got only one worker, stealing shouldn't be possible.
|
|
// Assume that this is a steal or return from spawn in a force-reduce case.
|
|
// We don't have a scheduling stack to switch to, so call the continuation
|
|
// function directly.
|
|
if (1 == w->g->P) {
|
|
fcn(w, ff, sf);
|
|
|
|
/* The call to function c() will have pushed ff as the next frame. If
|
|
* this were a normal (non-forced-reduce) execution, there would have
|
|
* been a pop_next_frame call in a separate part of the runtime. We
|
|
* must call pop_next_frame here to complete the push/pop cycle. */
|
|
ff2 = pop_next_frame(w);
|
|
|
|
setup_for_execution(w, ff2, 0);
|
|
scheduling_fiber_prepare_to_resume_user_code(w, ff2, w->current_stack_frame);
|
|
cilkrts_resume(w->current_stack_frame, ff2);
|
|
|
|
// Suppress clang warning that the expression result is unused
|
|
#if defined(__clang__) && (! defined(__INTEL_COMPILER))
|
|
# pragma clang diagnostic push
|
|
# pragma clang diagnostic ignored "-Wunused-value"
|
|
#endif // __clang__
|
|
/* no return */
|
|
CILK_ASSERT(((void)"returned from __cilkrts_resume", 0));
|
|
#if defined(__clang__) && (! defined(__INTEL_COMPILER))
|
|
# pragma clang diagnostic pop
|
|
#endif // __clang__
|
|
}
|
|
|
|
w->l->post_suspend = fcn;
|
|
w->l->suspended_stack = sf;
|
|
|
|
ITT_SYNC_RELEASING(w);
|
|
ITT_SYNC_PREPARE(w);
|
|
|
|
#if FIBER_DEBUG >= 2
|
|
fprintf(stderr, "ThreadId=%p, W=%d: about to switch into runtime... w->l->frame_ff = %p, sf=%p\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self, w->l->frame_ff,
|
|
sf);
|
|
#endif
|
|
|
|
// Current fiber is either the (1) one we are about to free,
|
|
// or (2) it has been passed up to the parent.
|
|
cilk_fiber *current_fiber = ( w->l->fiber_to_free ?
|
|
w->l->fiber_to_free :
|
|
w->l->frame_ff->parent->fiber_child );
|
|
cilk_fiber_data* fdata = cilk_fiber_get_data(current_fiber);
|
|
CILK_ASSERT(NULL == w->l->frame_ff->fiber_self);
|
|
|
|
// Clear the sf in the current fiber for cleanliness, to prevent
|
|
// us from accidentally resuming a bad sf.
|
|
// Technically, resume_sf gets overwritten for a fiber when
|
|
// we are about to resume it anyway.
|
|
fdata->resume_sf = NULL;
|
|
CILK_ASSERT(fdata->owner == w);
|
|
|
|
// Set the function to execute immediately after switching to the
|
|
// scheduling fiber, but before freeing any fibers.
|
|
cilk_fiber_set_post_switch_proc(w->l->scheduling_fiber,
|
|
enter_runtime_transition_proc);
|
|
cilk_fiber_invoke_tbb_stack_op(current_fiber, CILK_TBB_STACK_ORPHAN);
|
|
|
|
if (w->l->fiber_to_free) {
|
|
// Case 1: we are freeing this fiber. We never
|
|
// resume this fiber again after jumping into the runtime.
|
|
w->l->fiber_to_free = NULL;
|
|
|
|
// Extra check. Normally, the fiber we are about to switch to
|
|
// should have a NULL owner.
|
|
CILK_ASSERT(NULL == cilk_fiber_get_data(w->l->scheduling_fiber)->owner);
|
|
#if FIBER_DEBUG >= 4
|
|
fprintf(stderr, "ThreadId=%p, W=%d: about to switch into runtime.. current_fiber = %p, deallcoate, switch to fiber %p\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self,
|
|
current_fiber, w->l->scheduling_fiber);
|
|
#endif
|
|
cilk_fiber_invoke_tbb_stack_op(current_fiber, CILK_TBB_STACK_RELEASE);
|
|
NOTE_INTERVAL(w, INTERVAL_DEALLOCATE_RESUME_OTHER);
|
|
cilk_fiber_remove_reference_from_self_and_resume_other(current_fiber,
|
|
&w->l->fiber_pool,
|
|
w->l->scheduling_fiber);
|
|
// We should never come back here!
|
|
CILK_ASSERT(0);
|
|
}
|
|
else {
|
|
// Case 2: We are passing the fiber to our parent because we
|
|
// are leftmost. We should come back later to
|
|
// resume execution of user code.
|
|
//
|
|
// If we are not freeing a fiber, there we must be
|
|
// returning from a spawn or processing an exception. The
|
|
// "sync" path always frees a fiber.
|
|
//
|
|
// We must be the leftmost child, and by left holder logic, we
|
|
// have already moved the current fiber into our parent full
|
|
// frame.
|
|
#if FIBER_DEBUG >= 2
|
|
fprintf(stderr, "ThreadId=%p, W=%d: about to suspend self into runtime.. current_fiber = %p, deallcoate, switch to fiber %p\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self,
|
|
current_fiber, w->l->scheduling_fiber);
|
|
#endif
|
|
|
|
NOTE_INTERVAL(w, INTERVAL_SUSPEND_RESUME_OTHER);
|
|
|
|
cilk_fiber_suspend_self_and_resume_other(current_fiber,
|
|
w->l->scheduling_fiber);
|
|
// Resuming this fiber returns control back to
|
|
// this function because our implementation uses OS fibers.
|
|
//
|
|
// On Unix, we could have the choice of passing the
|
|
// user_code_resume_after_switch_into_runtime as an extra "resume_proc"
|
|
// that resumes execution of user code instead of the
|
|
// jumping back here, and then jumping back to user code.
|
|
#if FIBER_DEBUG >= 2
|
|
CILK_ASSERT(fdata->owner == __cilkrts_get_tls_worker());
|
|
#endif
|
|
user_code_resume_after_switch_into_runtime(current_fiber);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Send a message to the children of the specified worker: run or wait.
|
|
*/
|
|
static void notify_children(__cilkrts_worker *w, unsigned int msg)
|
|
{
|
|
int child_num;
|
|
__cilkrts_worker *child;
|
|
int num_sys_workers = w->g->P - 1;
|
|
|
|
// If worker is "n", then its children are 2n + 1, and 2n + 2.
|
|
child_num = (w->self << 1) + 1;
|
|
if (child_num < num_sys_workers) {
|
|
child = w->g->workers[child_num];
|
|
CILK_ASSERT(child->l->signal_node);
|
|
signal_node_msg(child->l->signal_node, msg);
|
|
child_num++;
|
|
if (child_num < num_sys_workers) {
|
|
child = w->g->workers[child_num];
|
|
CILK_ASSERT(child->l->signal_node);
|
|
signal_node_msg(child->l->signal_node, msg);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Notify this worker's children that they need to wait.
|
|
*/
|
|
static void notify_children_wait(__cilkrts_worker *w)
|
|
{
|
|
notify_children(w, 0);
|
|
}
|
|
|
|
/*
|
|
* Notify this worker's children to run and start trying to steal.
|
|
*/
|
|
static void notify_children_run(__cilkrts_worker *w)
|
|
{
|
|
notify_children(w, 1);
|
|
}
|
|
|
|
/**
|
|
* A single "check" to find work, either on our queue or through a
|
|
* steal attempt. This method checks our local queue once, and
|
|
* performs one steal attempt.
|
|
*/
|
|
static full_frame* check_for_work(__cilkrts_worker *w)
|
|
{
|
|
full_frame *ff = NULL;
|
|
ff = pop_next_frame(w);
|
|
// If there is no work on the queue, try to steal some.
|
|
if (NULL == ff) {
|
|
START_INTERVAL(w, INTERVAL_STEALING) {
|
|
if (w->l->type != WORKER_USER && w->l->team != NULL) {
|
|
// At this point, the worker knows for certain that it has run
|
|
// out of work. Therefore, it loses its team affiliation. User
|
|
// workers never change teams, of course.
|
|
__cilkrts_worker_lock(w);
|
|
w->l->team = NULL;
|
|
__cilkrts_worker_unlock(w);
|
|
}
|
|
|
|
// If we are about to do a random steal, we should have no
|
|
// full frame...
|
|
CILK_ASSERT(NULL == w->l->frame_ff);
|
|
random_steal(w);
|
|
} STOP_INTERVAL(w, INTERVAL_STEALING);
|
|
|
|
// If the steal was successful, then the worker has populated its next
|
|
// frame with the work to resume.
|
|
ff = pop_next_frame(w);
|
|
if (NULL == ff) {
|
|
// Punish the worker for failing to steal.
|
|
// No quantum for you!
|
|
__cilkrts_yield();
|
|
w->l->steal_failure_count++;
|
|
} else {
|
|
// Reset steal_failure_count since there is obviously still work to
|
|
// be done.
|
|
w->l->steal_failure_count = 0;
|
|
}
|
|
}
|
|
return ff;
|
|
}
|
|
|
|
/**
|
|
* Keep stealing or looking on our queue.
|
|
*
|
|
* Returns either when a full frame is found, or NULL if the
|
|
* computation is done.
|
|
*/
|
|
static full_frame* search_until_work_found_or_done(__cilkrts_worker *w)
|
|
{
|
|
full_frame *ff = NULL;
|
|
// Find a full frame to execute (either through random stealing,
|
|
// or because we pull it off w's 1-element queue).
|
|
while (!ff) {
|
|
// Check worker state to figure out our next action.
|
|
switch (worker_runnable(w))
|
|
{
|
|
case SCHEDULE_RUN: // One attempt at checking for work.
|
|
ff = check_for_work(w);
|
|
break;
|
|
case SCHEDULE_WAIT: // go into wait-mode.
|
|
CILK_ASSERT(WORKER_SYSTEM == w->l->type);
|
|
// If we are about to wait, then we better not have
|
|
// a frame that we should execute...
|
|
CILK_ASSERT(NULL == w->l->next_frame_ff);
|
|
notify_children_wait(w);
|
|
signal_node_wait(w->l->signal_node);
|
|
// ...
|
|
// Runtime is waking up.
|
|
notify_children_run(w);
|
|
w->l->steal_failure_count = 0;
|
|
break;
|
|
case SCHEDULE_EXIT: // exit the scheduler.
|
|
CILK_ASSERT(WORKER_USER != w->l->type);
|
|
return NULL;
|
|
default:
|
|
CILK_ASSERT(0);
|
|
abort();
|
|
}
|
|
}
|
|
return ff;
|
|
}
|
|
|
|
/**
|
|
* The proc method for a scheduling fiber on a user worker.
|
|
*
|
|
* When a user worker jumps into the runtime, it jumps into this
|
|
* method by either starting it if the scheduling fiber has never run
|
|
* before, or resuming the fiber if it was previously suspended.
|
|
*/
|
|
COMMON_PORTABLE
|
|
void scheduler_fiber_proc_for_user_worker(cilk_fiber *fiber)
|
|
{
|
|
__cilkrts_worker* w = cilk_fiber_get_owner(fiber);
|
|
CILK_ASSERT(w);
|
|
|
|
// This must be a user worker
|
|
CILK_ASSERT(WORKER_USER == w->l->type);
|
|
|
|
// If we aren't the current worker, then something is very wrong
|
|
// here..
|
|
verify_current_wkr(w);
|
|
|
|
__cilkrts_run_scheduler_with_exceptions(w);
|
|
}
|
|
|
|
|
|
/**
|
|
* The body of the runtime scheduling loop. This function executes in
|
|
* 4 stages:
|
|
*
|
|
* 1. Transitions from the user code into the runtime by
|
|
* executing any scheduling-stack functions.
|
|
* 2. Looks for a full frame enqueued from a successful provably
|
|
* good steal.
|
|
* 3. If no full frame is found in step 2, steal until
|
|
* a frame is found or we are done. If we are done, finish
|
|
* the scheduling loop.
|
|
* 4. When a frame is found, setup to resume user code.
|
|
* In particular, suspend the current fiber and resume the
|
|
* user fiber to execute the frame.
|
|
*
|
|
* Returns a fiber object that we should switch to after completing
|
|
* the body of the loop, or NULL if we should continue executing on
|
|
* this fiber.
|
|
*
|
|
* @pre @c current_fiber should equal @c wptr->l->scheduling_fiber
|
|
*
|
|
* @param current_fiber The currently executing (scheduling_ fiber
|
|
* @param wptr The currently executing worker.
|
|
* @param return The next fiber we should switch to.
|
|
*/
|
|
static cilk_fiber* worker_scheduling_loop_body(cilk_fiber* current_fiber,
|
|
void* wptr)
|
|
{
|
|
__cilkrts_worker *w = (__cilkrts_worker*) wptr;
|
|
CILK_ASSERT(current_fiber == w->l->scheduling_fiber);
|
|
|
|
// Stage 1: Transition from executing user code to the runtime code.
|
|
// We don't need to do this call here any more, because
|
|
// every switch to the scheduling fiber should make this call
|
|
// using a post_switch_proc on the fiber.
|
|
//
|
|
// enter_runtime_transition_proc(w->l->scheduling_fiber, wptr);
|
|
|
|
// After Stage 1 is complete, w should no longer have
|
|
// an associated full frame.
|
|
CILK_ASSERT(NULL == w->l->frame_ff);
|
|
|
|
// Stage 2. First do a quick check of our 1-element queue.
|
|
full_frame *ff = pop_next_frame(w);
|
|
|
|
if (!ff) {
|
|
// Stage 3. We didn't find anything from our 1-element
|
|
// queue. Now go through the steal loop to find work.
|
|
ff = search_until_work_found_or_done(w);
|
|
if (!ff) {
|
|
CILK_ASSERT(w->g->work_done);
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
// Stage 4. Now that we have found a full frame to work on,
|
|
// actually execute it.
|
|
__cilkrts_stack_frame *sf;
|
|
|
|
// There shouldn't be any uncaught exceptions.
|
|
//
|
|
// In Windows, the OS catches any exceptions not caught by the
|
|
// user code. Thus, we are omitting the check on Windows.
|
|
//
|
|
// On Android, calling std::uncaught_exception with the stlport
|
|
// library causes a seg fault. Since we're not supporting
|
|
// exceptions there at this point, just don't do the check
|
|
CILKBUG_ASSERT_NO_UNCAUGHT_EXCEPTION();
|
|
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
CILK_ASSERT(!w->l->frame_ff);
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
sf = ff->call_stack;
|
|
CILK_ASSERT(sf && !sf->call_parent);
|
|
setup_for_execution(w, ff, 0);
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
} END_WITH_WORKER_LOCK(w);
|
|
|
|
/* run it */
|
|
//
|
|
// Prepare to run the full frame. To do so, we need to:
|
|
// (a) Execute some code on this fiber (the scheduling
|
|
// fiber) to set up data structures, and
|
|
// (b) Suspend the scheduling fiber, and resume the
|
|
// user-code fiber.
|
|
|
|
// Part (a). Set up data structures.
|
|
scheduling_fiber_prepare_to_resume_user_code(w, ff, sf);
|
|
|
|
cilk_fiber *other = w->l->frame_ff->fiber_self;
|
|
cilk_fiber_data* other_data = cilk_fiber_get_data(other);
|
|
cilk_fiber_data* current_fiber_data = cilk_fiber_get_data(current_fiber);
|
|
|
|
// I believe two cases are possible here, both of which
|
|
// should have other_data->resume_sf as NULL.
|
|
//
|
|
// 1. Resuming a fiber that was previously executing
|
|
// user code (i.e., a provably-good-steal).
|
|
// In this case, resume_sf should have been
|
|
// set to NULL when it was suspended.
|
|
//
|
|
// 2. Resuming code on a steal. In this case, since we
|
|
// grabbed a new fiber, resume_sf should be NULL.
|
|
CILK_ASSERT(NULL == other_data->resume_sf);
|
|
|
|
#if FIBER_DEBUG >= 2
|
|
fprintf(stderr, "W=%d: other fiber=%p, setting resume_sf to %p\n",
|
|
w->self, other, other_data->resume_sf);
|
|
#endif
|
|
// Update our own fiber's data.
|
|
current_fiber_data->resume_sf = NULL;
|
|
// The scheduling fiber should have the right owner from before.
|
|
CILK_ASSERT(current_fiber_data->owner == w);
|
|
other_data->resume_sf = sf;
|
|
|
|
|
|
#if FIBER_DEBUG >= 3
|
|
fprintf(stderr, "ThreadId=%p (about to suspend self resume other), W=%d: current_fiber=%p, other=%p, current_fiber->resume_sf = %p, other->resume_sf = %p\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self,
|
|
current_fiber, other,
|
|
current_fiber_data->resume_sf,
|
|
other_data->resume_sf);
|
|
#endif
|
|
return other;
|
|
}
|
|
|
|
|
|
/**
|
|
* This function is executed once by each worker, to initialize its
|
|
* scheduling loop.
|
|
*/
|
|
static void worker_scheduler_init_function(__cilkrts_worker *w)
|
|
{
|
|
// First, execute the startup tasks that must happen for all
|
|
// worker types.
|
|
ITT_SYNC_PREPARE(w);
|
|
/* Notify tools about the new worker. Inspector needs this, but we
|
|
don't want to confuse Cilkscreen with system threads. User threads
|
|
do this notification in bind_thread */
|
|
if (! w->g->under_ptool)
|
|
__cilkrts_cilkscreen_establish_worker(w);
|
|
|
|
// Seed the initial random number generator.
|
|
// If we forget to do this, then the worker always steals from 0.
|
|
// Programs will still execute correctly, but
|
|
// you may see a subtle performance bug...
|
|
mysrand(w, (w->self + 1));
|
|
|
|
// The startup work varies, depending on the worker type.
|
|
switch (w->l->type) {
|
|
case WORKER_USER:
|
|
// Stop working once we've entered the scheduler.
|
|
// For user workers, INTERVAL_IN_SCHEDULER counts the time
|
|
// since we called bind_thread.
|
|
break;
|
|
|
|
case WORKER_SYSTEM:
|
|
// If a system worker is starting, we must also be starting
|
|
// the runtime.
|
|
|
|
// Runtime begins in a wait-state and is woken up by the first user
|
|
// worker when the runtime is ready.
|
|
signal_node_wait(w->l->signal_node);
|
|
// ...
|
|
// Runtime is waking up.
|
|
notify_children_run(w);
|
|
w->l->steal_failure_count = 0;
|
|
|
|
// For system threads, count all the time this thread is
|
|
// alive in the scheduling loop.
|
|
START_INTERVAL(w, INTERVAL_IN_SCHEDULER);
|
|
START_INTERVAL(w, INTERVAL_WORKING);
|
|
break;
|
|
default:
|
|
__cilkrts_bug("Unknown worker %p of type %d entering scheduling loop\n",
|
|
w, w->l->type);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This function is executed once by each worker, to finish its
|
|
* scheduling loop.
|
|
*
|
|
* @note Currently, only system workers finish their loops. User
|
|
* workers will jump away to user code without exiting their
|
|
* scheduling loop.
|
|
*/
|
|
static void worker_scheduler_terminate_function(__cilkrts_worker *w)
|
|
{
|
|
// A user worker should never finish by falling through the
|
|
// scheduling loop.
|
|
CILK_ASSERT(WORKER_USER != w->l->type);
|
|
STOP_INTERVAL(w, INTERVAL_IN_RUNTIME);
|
|
STOP_INTERVAL(w, INTERVAL_IN_SCHEDULER);
|
|
}
|
|
|
|
/**
|
|
* The main scheduler function executed by a worker's scheduling
|
|
* fiber.
|
|
*
|
|
* This method is started by either a new system worker, or a user
|
|
* worker that has stalled and just been imported into the runtime.
|
|
*/
|
|
static void worker_scheduler_function(__cilkrts_worker *w)
|
|
{
|
|
worker_scheduler_init_function(w);
|
|
|
|
// The main scheduling loop body.
|
|
|
|
while (!w->g->work_done) {
|
|
// Set intervals. Now we are in the runtime instead of working.
|
|
START_INTERVAL(w, INTERVAL_IN_RUNTIME);
|
|
STOP_INTERVAL(w, INTERVAL_WORKING);
|
|
|
|
// Execute the "body" of the scheduling loop, and figure
|
|
// out the fiber to jump to next.
|
|
cilk_fiber* fiber_to_resume
|
|
= worker_scheduling_loop_body(w->l->scheduling_fiber, w);
|
|
|
|
if (fiber_to_resume) {
|
|
// Suspend the current fiber and resume next one.
|
|
NOTE_INTERVAL(w, INTERVAL_SUSPEND_RESUME_OTHER);
|
|
STOP_INTERVAL(w, INTERVAL_IN_RUNTIME);
|
|
START_INTERVAL(w, INTERVAL_WORKING);
|
|
cilk_fiber_suspend_self_and_resume_other(w->l->scheduling_fiber,
|
|
fiber_to_resume);
|
|
|
|
// Return here only when this (scheduling) fiber is
|
|
// resumed (i.e., this worker wants to reenter the runtime).
|
|
}
|
|
}
|
|
|
|
// Finish the scheduling loop.
|
|
worker_scheduler_terminate_function(w);
|
|
}
|
|
|
|
|
|
/*************************************************************
|
|
Forward declarations for reduction protocol.
|
|
*************************************************************/
|
|
|
|
static __cilkrts_worker*
|
|
execute_reductions_for_sync(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf_at_sync);
|
|
|
|
static __cilkrts_worker*
|
|
execute_reductions_for_spawn_return(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *returning_sf);
|
|
|
|
|
|
|
|
/*************************************************************
|
|
Scheduler functions that are callable by client code
|
|
*************************************************************/
|
|
static full_frame *disown(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf,
|
|
const char *why)
|
|
{
|
|
CILK_ASSERT(ff);
|
|
make_unrunnable(w, ff, sf, sf != 0, why);
|
|
w->l->frame_ff = 0;
|
|
return ff->parent;
|
|
}
|
|
|
|
/**
|
|
* Called when ff is returning from a spawn, and we need to execute a
|
|
* reduction.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame for w.
|
|
* @param returning_sf The stack frame for the spawn helper that is returning.
|
|
*
|
|
* Normally, by the time we gain control in the runtime, the worker
|
|
* has already popped off the __cilkrts_stack_frame "returning_sf"
|
|
* from its call chain.
|
|
*
|
|
* When we have only serial reductions, w->current_stack_frame is not
|
|
* needed any more, because w is about to enter the runtime scheduling
|
|
* loop anyway. Similarly, the frame "ff" is slated to be destroyed
|
|
* after the runtime finishes the return from spawn and splices ff out
|
|
* of the tree of full frames.
|
|
*
|
|
* To execute a parallel reduction, however, we still want
|
|
* w->current_stack_frame == returning_sf, and we are going to use the
|
|
* frame ff for a little bit longer.
|
|
*
|
|
* This method:
|
|
*
|
|
* 1. Puts returning_sf back as w's current stack frame.
|
|
* 2. Makes "ff" runnable again on w.
|
|
*/
|
|
static inline
|
|
void restore_frame_for_spawn_return_reduction(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *returning_sf) {
|
|
#if REDPAR_DEBUG >= 2
|
|
CILK_ASSERT(returning_sf);
|
|
CILK_ASSERT(returning_sf->worker == w);
|
|
#endif
|
|
// Change w's current stack frame back to "returning_sf".
|
|
//
|
|
// Intuitively, w->current_stack_frame should be
|
|
// returning_sf->call_parent at this point.
|
|
//
|
|
// We can not assert this, however, because the pop of
|
|
// returning_sf from the call chain has already cleared
|
|
// returning_sf->call_parent. We don't want to restore the call
|
|
// parent of returning_sf, because its parent has been stolen, and
|
|
// the runtime assumes that steals break this link.
|
|
|
|
// We cannot assert call_parent is NULL either, since that's not true for
|
|
// Win64 exception handling
|
|
// CILK_ASSERT(returning_sf->call_parent == NULL);
|
|
w->current_stack_frame = returning_sf;
|
|
|
|
// Make the full frame "ff" runnable again, in preparation for
|
|
// executing the reduction.
|
|
make_runnable(w, ff);
|
|
}
|
|
|
|
|
|
NORETURN __cilkrts_c_sync(__cilkrts_worker *w,
|
|
__cilkrts_stack_frame *sf_at_sync)
|
|
{
|
|
full_frame *ff;
|
|
|
|
// Claim: This read of w->l->frame_ff can occur without
|
|
// holding the worker lock because when w has reached a sync
|
|
// and entered the runtime (because it stalls), w's deque is empty
|
|
// and no one else can steal and change w->l->frame_ff.
|
|
|
|
ff = w->l->frame_ff;
|
|
#ifdef _WIN32
|
|
__cilkrts_save_exception_state(w, ff);
|
|
#else
|
|
// Move any pending exceptions into the full frame
|
|
CILK_ASSERT(NULL == ff->pending_exception);
|
|
ff->pending_exception = w->l->pending_exception;
|
|
w->l->pending_exception = NULL;
|
|
#endif
|
|
|
|
w = execute_reductions_for_sync(w, ff, sf_at_sync);
|
|
|
|
#if FIBER_DEBUG >= 3
|
|
fprintf(stderr, "ThreadId=%p, w->self = %d. about to longjmp_into_runtim[c_sync] with ff=%p\n",
|
|
cilkos_get_current_thread_id(), w->self, ff);
|
|
#endif
|
|
|
|
longjmp_into_runtime(w, do_sync, sf_at_sync);
|
|
}
|
|
|
|
static void do_sync(__cilkrts_worker *w, full_frame *ff,
|
|
__cilkrts_stack_frame *sf)
|
|
{
|
|
//int abandoned = 1;
|
|
enum provably_good_steal_t steal_result = ABANDON_EXECUTION;
|
|
|
|
START_INTERVAL(w, INTERVAL_SYNC_CHECK) {
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) {
|
|
|
|
CILK_ASSERT(ff);
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
CILK_ASSERT(sf->call_parent == 0);
|
|
CILK_ASSERT(sf->flags & CILK_FRAME_UNSYNCHED);
|
|
|
|
// Before switching into the scheduling fiber, we should have
|
|
// already taken care of deallocating the current
|
|
// fiber.
|
|
CILK_ASSERT(NULL == ff->fiber_self);
|
|
|
|
// Update the frame's pedigree information if this is an ABI 1
|
|
// or later frame
|
|
if (CILK_FRAME_VERSION_VALUE(sf->flags) >= 1)
|
|
{
|
|
sf->parent_pedigree.rank = w->pedigree.rank;
|
|
sf->parent_pedigree.parent = w->pedigree.parent;
|
|
|
|
// Note that the pedigree rank needs to be updated
|
|
// when setup_for_execution_pedigree runs
|
|
sf->flags |= CILK_FRAME_SF_PEDIGREE_UNSYNCHED;
|
|
}
|
|
|
|
/* the decjoin() occurs in provably_good_steal() */
|
|
steal_result = provably_good_steal(w, ff);
|
|
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
// set w->l->frame_ff = NULL after checking abandoned
|
|
if (WAIT_FOR_CONTINUE != steal_result) {
|
|
w->l->frame_ff = NULL;
|
|
}
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
} STOP_INTERVAL(w, INTERVAL_SYNC_CHECK);
|
|
|
|
// Now, if we are in a replay situation and provably_good_steal() returned
|
|
// WAIT_FOR_CONTINUE, we should sleep, reacquire locks, call
|
|
// provably_good_steal(), and release locks until we get a value other
|
|
// than WAIT_FOR_CONTINUE from the function.
|
|
#ifdef CILK_RECORD_REPLAY
|
|
// We don't have to explicitly check for REPLAY_LOG below because
|
|
// steal_result can only be set to WAIT_FOR_CONTINUE during replay
|
|
while(WAIT_FOR_CONTINUE == steal_result)
|
|
{
|
|
__cilkrts_sleep();
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w)
|
|
{
|
|
ff = w->l->frame_ff;
|
|
BEGIN_WITH_FRAME_LOCK(w, ff)
|
|
{
|
|
steal_result = provably_good_steal(w, ff);
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
if (WAIT_FOR_CONTINUE != steal_result)
|
|
w->l->frame_ff = NULL;
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
}
|
|
#endif // CILK_RECORD_REPLAY
|
|
|
|
#ifdef ENABLE_NOTIFY_ZC_INTRINSIC
|
|
// If we can't make any further progress on this thread, tell Inspector
|
|
// that we're abandoning the work and will go find something else to do.
|
|
if (ABANDON_EXECUTION == steal_result)
|
|
{
|
|
NOTIFY_ZC_INTRINSIC("cilk_sync_abandon", 0);
|
|
}
|
|
#endif // defined ENABLE_NOTIFY_ZC_INTRINSIC
|
|
|
|
return; /* back to scheduler loop */
|
|
}
|
|
|
|
/* worker W completely promotes its own deque, simulating the case
|
|
where the whole deque is stolen. We use this mechanism to force
|
|
the allocation of new storage for reducers for race-detection
|
|
purposes. */
|
|
void __cilkrts_promote_own_deque(__cilkrts_worker *w)
|
|
{
|
|
// Remember the fiber we start this method on.
|
|
CILK_ASSERT(w->l->frame_ff);
|
|
cilk_fiber* starting_fiber = w->l->frame_ff->fiber_self;
|
|
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
while (dekker_protocol(w)) {
|
|
/* PLACEHOLDER_FIBER is used as non-null marker to tell detach()
|
|
and make_child() that this frame should be treated as a spawn
|
|
parent, even though we have not assigned it a stack. */
|
|
detach_for_steal(w, w, PLACEHOLDER_FIBER);
|
|
}
|
|
} END_WITH_WORKER_LOCK(w);
|
|
|
|
|
|
// TBD: The management of full frames and fibers is a bit
|
|
// sketchy here. We are promoting stack frames into full frames,
|
|
// and pretending they are stolen away, but no other worker is
|
|
// actually working on them. Some runtime invariants
|
|
// may be broken here.
|
|
//
|
|
// Technically, if we are simulating a steal from w
|
|
// w should get a new full frame, but
|
|
// keep the same fiber. A real thief would be taking the
|
|
// loot frame away, get a new fiber, and starting executing the
|
|
// loot frame.
|
|
//
|
|
// What should a fake thief do? Where does the frame go?
|
|
|
|
// In any case, we should be finishing the promotion process with
|
|
// the same fiber with.
|
|
CILK_ASSERT(w->l->frame_ff);
|
|
CILK_ASSERT(w->l->frame_ff->fiber_self == starting_fiber);
|
|
}
|
|
|
|
|
|
|
|
/* the client code calls this function after a spawn when the dekker
|
|
protocol fails. The function may either return or longjmp
|
|
into the rts
|
|
|
|
This function takes in a "returning_sf" argument which corresponds
|
|
to the __cilkrts_stack_frame that we are finishing (i.e., the
|
|
argument to __cilkrts_leave_frame).
|
|
*/
|
|
void __cilkrts_c_THE_exception_check(__cilkrts_worker *w,
|
|
__cilkrts_stack_frame *returning_sf)
|
|
{
|
|
full_frame *ff;
|
|
int stolen_p;
|
|
__cilkrts_stack_frame *saved_sf = NULL;
|
|
|
|
START_INTERVAL(w, INTERVAL_THE_EXCEPTION_CHECK);
|
|
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
ff = w->l->frame_ff;
|
|
CILK_ASSERT(ff);
|
|
/* This code is called only upon a normal return and never
|
|
upon an exceptional return. Assert that this is the
|
|
case. */
|
|
CILK_ASSERT(!w->l->pending_exception);
|
|
|
|
reset_THE_exception(w);
|
|
stolen_p = !(w->head < (w->tail + 1)); /* +1 because tail was
|
|
speculatively
|
|
decremented by the
|
|
compiled code */
|
|
|
|
if (stolen_p) {
|
|
/* XXX This will be charged to THE for accounting purposes */
|
|
__cilkrts_save_exception_state(w, ff);
|
|
|
|
// Save the value of the current stack frame.
|
|
saved_sf = w->current_stack_frame;
|
|
|
|
// Reverse the decrement from undo_detach.
|
|
// This update effectively resets the deque to be
|
|
// empty (i.e., changes w->tail back to equal w->head).
|
|
// We need to reset the deque to execute parallel
|
|
// reductions. When we have only serial reductions, it
|
|
// does not matter, since serial reductions do not
|
|
// change the deque.
|
|
w->tail++;
|
|
#if REDPAR_DEBUG > 1
|
|
// ASSERT our deque is empty.
|
|
CILK_ASSERT(w->head == w->tail);
|
|
#endif
|
|
}
|
|
} END_WITH_WORKER_LOCK(w);
|
|
|
|
STOP_INTERVAL(w, INTERVAL_THE_EXCEPTION_CHECK);
|
|
|
|
if (stolen_p)
|
|
{
|
|
w = execute_reductions_for_spawn_return(w, ff, returning_sf);
|
|
|
|
// "Mr. Policeman? My parent always told me that if I was in trouble
|
|
// I should ask a nice policeman for help. I can't find my parent
|
|
// anywhere..."
|
|
//
|
|
// Write a record to the replay log for an attempt to return to a stolen parent
|
|
replay_record_orphaned(w);
|
|
|
|
// Update the pedigree only after we've finished the
|
|
// reductions.
|
|
update_pedigree_on_leave_frame(w, returning_sf);
|
|
|
|
// Notify Inspector that the parent has been stolen and we're
|
|
// going to abandon this work and go do something else. This
|
|
// will match the cilk_leave_begin in the compiled code
|
|
NOTIFY_ZC_INTRINSIC("cilk_leave_stolen", saved_sf);
|
|
|
|
DBGPRINTF ("%d: longjmp_into_runtime from __cilkrts_c_THE_exception_check\n", w->self);
|
|
longjmp_into_runtime(w, do_return_from_spawn, 0);
|
|
DBGPRINTF ("%d: returned from longjmp_into_runtime from __cilkrts_c_THE_exception_check?!\n", w->self);
|
|
}
|
|
else
|
|
{
|
|
NOTE_INTERVAL(w, INTERVAL_THE_EXCEPTION_CHECK_USELESS);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/* Return an exception to a stolen parent. */
|
|
NORETURN __cilkrts_exception_from_spawn(__cilkrts_worker *w,
|
|
__cilkrts_stack_frame *returning_sf)
|
|
{
|
|
full_frame *ff = w->l->frame_ff;
|
|
// This is almost the same as THE_exception_check, except
|
|
// the detach didn't happen, we don't need to undo the tail
|
|
// update.
|
|
CILK_ASSERT(w->head == w->tail);
|
|
w = execute_reductions_for_spawn_return(w, ff, returning_sf);
|
|
|
|
longjmp_into_runtime(w, do_return_from_spawn, 0);
|
|
CILK_ASSERT(0);
|
|
}
|
|
|
|
static void do_return_from_spawn(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf)
|
|
{
|
|
full_frame *parent_ff;
|
|
enum provably_good_steal_t steal_result = ABANDON_EXECUTION;
|
|
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) {
|
|
CILK_ASSERT(ff);
|
|
CILK_ASSERT(!ff->is_call_child);
|
|
CILK_ASSERT(sf == NULL);
|
|
parent_ff = ff->parent;
|
|
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
decjoin(ff);
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
|
|
BEGIN_WITH_FRAME_LOCK(w, parent_ff) {
|
|
if (parent_ff->simulated_stolen)
|
|
unconditional_steal(w, parent_ff);
|
|
else
|
|
steal_result = provably_good_steal(w, parent_ff);
|
|
} END_WITH_FRAME_LOCK(w, parent_ff);
|
|
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
|
|
// Loop here in replay mode
|
|
#ifdef CILK_RECORD_REPLAY
|
|
// We don't have to explicitly check for REPLAY_LOG below because
|
|
// steal_result can only get set to WAIT_FOR_CONTINUE during replay.
|
|
// We also don't have to worry about the simulated_stolen flag
|
|
// because steal_result can only be set to WAIT_FOR_CONTINUE by
|
|
// provably_good_steal().
|
|
while(WAIT_FOR_CONTINUE == steal_result)
|
|
{
|
|
__cilkrts_sleep();
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w)
|
|
{
|
|
BEGIN_WITH_FRAME_LOCK(w, parent_ff)
|
|
{
|
|
steal_result = provably_good_steal(w, parent_ff);
|
|
} END_WITH_FRAME_LOCK(w, parent_ff);
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
}
|
|
#endif // CILK_RECORD_REPLAY
|
|
|
|
// Cleanup the child frame.
|
|
__cilkrts_destroy_full_frame(w, ff);
|
|
return;
|
|
}
|
|
|
|
#ifdef _WIN32
|
|
/* migrate an exception across fibers. Call this function when an exception has
|
|
* been thrown and has to traverse across a steal. The exception has already
|
|
* been wrapped up, so all that remains is to longjmp() into the continuation,
|
|
* sync, and re-raise it.
|
|
*/
|
|
void __cilkrts_migrate_exception(__cilkrts_stack_frame *sf) {
|
|
|
|
__cilkrts_worker *w = sf->worker;
|
|
full_frame *ff;
|
|
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
ff = w->l->frame_ff;
|
|
reset_THE_exception(w);
|
|
/* there is no need to check for a steal because we wouldn't be here if
|
|
there weren't a steal. */
|
|
__cilkrts_save_exception_state(w, ff);
|
|
|
|
CILK_ASSERT(w->head == w->tail);
|
|
} END_WITH_WORKER_LOCK(w);
|
|
|
|
{
|
|
// TBD(jsukha): This function emulates the
|
|
// the "do_return_from_spawn" path.
|
|
w = execute_reductions_for_spawn_return(w, ff, sf);
|
|
}
|
|
|
|
longjmp_into_runtime(w, do_return_from_spawn, 0); /* does not return. */
|
|
CILK_ASSERT(! "Shouldn't be here...");
|
|
}
|
|
#endif
|
|
|
|
|
|
/* Pop a call stack from TAIL. Return the call stack, or NULL if the
|
|
queue is empty */
|
|
__cilkrts_stack_frame *__cilkrts_pop_tail(__cilkrts_worker *w)
|
|
{
|
|
__cilkrts_stack_frame *sf;
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
__cilkrts_stack_frame *volatile *tail = w->tail;
|
|
if (w->head < tail) {
|
|
--tail;
|
|
sf = *tail;
|
|
w->tail = tail;
|
|
} else {
|
|
sf = 0;
|
|
}
|
|
} END_WITH_WORKER_LOCK(w);
|
|
return sf;
|
|
}
|
|
|
|
#ifdef CILK_RECORD_REPLAY
|
|
__cilkrts_stack_frame *simulate_pop_tail(__cilkrts_worker *w)
|
|
{
|
|
__cilkrts_stack_frame *sf;
|
|
BEGIN_WITH_WORKER_LOCK(w) {
|
|
if (w->head < w->tail) {
|
|
sf = *(w->tail-1);
|
|
} else {
|
|
sf = 0;
|
|
}
|
|
} END_WITH_WORKER_LOCK(w);
|
|
return sf;
|
|
}
|
|
#endif
|
|
|
|
|
|
/* Return from a call, not a spawn. */
|
|
void __cilkrts_return(__cilkrts_worker *w)
|
|
{
|
|
full_frame *ff, *parent_ff;
|
|
START_INTERVAL(w, INTERVAL_RETURNING);
|
|
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) {
|
|
ff = w->l->frame_ff;
|
|
CILK_ASSERT(ff);
|
|
CILK_ASSERT(ff->join_counter == 1);
|
|
/* This path is not used to return from spawn. */
|
|
CILK_ASSERT(ff->is_call_child);
|
|
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
// After this call, w->l->frame_ff != ff.
|
|
// Technically, w will "own" ff until ff is freed,
|
|
// however, because ff is a dying leaf full frame.
|
|
parent_ff = disown(w, ff, 0, "return");
|
|
decjoin(ff);
|
|
|
|
#ifdef _WIN32
|
|
__cilkrts_save_exception_state(w, ff);
|
|
#else
|
|
// Move the pending exceptions into the full frame
|
|
// This should always be NULL if this isn't a
|
|
// return with an exception
|
|
CILK_ASSERT(NULL == ff->pending_exception);
|
|
ff->pending_exception = w->l->pending_exception;
|
|
w->l->pending_exception = NULL;
|
|
#endif // _WIN32
|
|
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
|
|
__cilkrts_fence(); /* redundant */
|
|
|
|
CILK_ASSERT(parent_ff);
|
|
|
|
BEGIN_WITH_FRAME_LOCK(w, parent_ff) {
|
|
finalize_child_for_call(w, parent_ff, ff);
|
|
} END_WITH_FRAME_LOCK(w, parent_ff);
|
|
|
|
ff = pop_next_frame(w);
|
|
/* ff will be non-null except when the parent frame is owned
|
|
by another worker.
|
|
CILK_ASSERT(ff)
|
|
*/
|
|
CILK_ASSERT(!w->l->frame_ff);
|
|
if (ff) {
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
__cilkrts_stack_frame *sf = ff->call_stack;
|
|
CILK_ASSERT(sf && !sf->call_parent);
|
|
setup_for_execution(w, ff, 1);
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
}
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
|
|
STOP_INTERVAL(w, INTERVAL_RETURNING);
|
|
}
|
|
|
|
static void __cilkrts_unbind_thread()
|
|
{
|
|
int stop_cilkscreen = 0;
|
|
global_state_t *g;
|
|
|
|
// Take out the global OS mutex to protect accesses to the table of workers
|
|
global_os_mutex_lock();
|
|
|
|
if (cilkg_is_published()) {
|
|
__cilkrts_worker *w = __cilkrts_get_tls_worker();
|
|
if (w) {
|
|
g = w->g;
|
|
|
|
// If there's only 1 worker, the counts will be stopped in
|
|
// __cilkrts_scheduler
|
|
if (g->P > 1)
|
|
{
|
|
STOP_INTERVAL(w, INTERVAL_WORKING);
|
|
STOP_INTERVAL(w, INTERVAL_IN_SCHEDULER);
|
|
}
|
|
|
|
__cilkrts_set_tls_worker(0);
|
|
|
|
if (w->self == -1) {
|
|
// This worker is an overflow worker. I.e., it was created on-
|
|
// demand when the global pool ran out of workers.
|
|
destroy_worker(w);
|
|
__cilkrts_free(w);
|
|
} else {
|
|
// This is a normal user worker and needs to be counted by the
|
|
// global state for the purposes of throttling system workers.
|
|
w->l->type = WORKER_FREE;
|
|
__cilkrts_leave_cilk(g);
|
|
}
|
|
|
|
stop_cilkscreen = (0 == g->Q);
|
|
}
|
|
}
|
|
global_os_mutex_unlock();
|
|
|
|
/* Turn off Cilkscreen. This needs to be done when we are NOT holding the
|
|
* os mutex. */
|
|
if (stop_cilkscreen)
|
|
__cilkrts_cilkscreen_disable_instrumentation();
|
|
}
|
|
|
|
/* special return from the initial frame */
|
|
|
|
void __cilkrts_c_return_from_initial(__cilkrts_worker *w)
|
|
{
|
|
struct cilkred_map *rm;
|
|
|
|
/* This is only called on a user thread worker. */
|
|
CILK_ASSERT(w->l->type == WORKER_USER);
|
|
|
|
#if REDPAR_DEBUG >= 3
|
|
fprintf(stderr, "[W=%d, desc=cilkrts_c_return_from_initial, ff=%p]\n",
|
|
w->self, w->l->frame_ff);
|
|
#endif
|
|
|
|
BEGIN_WITH_WORKER_LOCK_OPTIONAL(w) {
|
|
full_frame *ff = w->l->frame_ff;
|
|
CILK_ASSERT(ff);
|
|
CILK_ASSERT(ff->join_counter == 1);
|
|
w->l->frame_ff = 0;
|
|
|
|
CILK_ASSERT(ff->fiber_self);
|
|
// Save any TBB interop data for the next time this thread enters Cilk
|
|
cilk_fiber_tbb_interop_save_info_from_stack(ff->fiber_self);
|
|
|
|
// Deallocate cilk_fiber that mapped to the user stack. The stack
|
|
// itself does not get deallocated (of course) but our data
|
|
// structure becomes divorced from it.
|
|
|
|
#if FIBER_DEBUG >= 1
|
|
fprintf(stderr, "ThreadId=%p: w=%d: We are about to deallocate ff->fiber_self = %p here. w->l->scheduling_fiber = %p. w->l->type = %d\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self,
|
|
ff->fiber_self,
|
|
w->l->scheduling_fiber,
|
|
w->l->type);
|
|
#endif
|
|
// The fiber in ff is a user-code fiber. The fiber in
|
|
// w->l->scheduling_fiber is a scheduling fiber. These fibers should
|
|
// never be equal. When a user worker returns (and will unbind), we
|
|
// should destroy only the fiber in ff. The scheduling fiber will be
|
|
// re-used.
|
|
|
|
CILK_ASSERT(ff->fiber_self != w->l->scheduling_fiber);
|
|
|
|
START_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE) {
|
|
// This fiber might not be deallocated here if there
|
|
// is a pending exception on Windows that refers
|
|
// to this fiber.
|
|
//
|
|
// First "suspend" the fiber, and then try to delete it.
|
|
cilk_fiber_deallocate_from_thread(ff->fiber_self);
|
|
} STOP_INTERVAL(w, INTERVAL_FIBER_DEALLOCATE);
|
|
ff->fiber_self = NULL;
|
|
|
|
/* Save reducer map into global_state object */
|
|
rm = w->reducer_map;
|
|
w->reducer_map = NULL;
|
|
|
|
#if REDPAR_DEBUG >= 3
|
|
fprintf(stderr, "W=%d, reducer_map_to_delete=%p, was in ff=%p\n",
|
|
w->self,
|
|
rm,
|
|
ff);
|
|
#endif
|
|
__cilkrts_destroy_full_frame(w, ff);
|
|
|
|
|
|
/* Work is never done. w->g->work_done = 1; __cilkrts_fence(); */
|
|
} END_WITH_WORKER_LOCK_OPTIONAL(w);
|
|
|
|
|
|
save_pedigree_leaf_from_user_worker(w);
|
|
|
|
// Workers can have NULL reducer maps now.
|
|
if (rm) {
|
|
__cilkrts_destroy_reducer_map(w, rm);
|
|
}
|
|
|
|
|
|
#if FIBER_DEBUG >= 1
|
|
__cilkrts_worker* tmp = w;
|
|
int tmp_id = w->self;
|
|
fprintf(stderr, "w=%d: We are about unbind thread (w= %p)\n",
|
|
w->self,
|
|
w);
|
|
#endif
|
|
|
|
w = NULL;
|
|
|
|
__cilkrts_unbind_thread();
|
|
|
|
#if FIBER_DEBUG >= 1
|
|
|
|
fprintf(stderr, "w=%p, %d: Finished unbind\n",
|
|
tmp, tmp_id);
|
|
#endif
|
|
|
|
/* Other workers will stop trying to steal if this was the last worker. */
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
/*
|
|
* __cilkrts_restore_stealing
|
|
*
|
|
* Restore the protected_tail to a previous state, possibly allowing frames
|
|
* to be stolen. The dekker_protocol has been extended to steal only if
|
|
* head+1 is < protected_tail.
|
|
*/
|
|
|
|
void __cilkrts_restore_stealing(
|
|
__cilkrts_worker *w,
|
|
__cilkrts_stack_frame *volatile *saved_protected_tail)
|
|
{
|
|
/* On most x86 this pair of operations would be slightly faster
|
|
as an atomic exchange due to the implicit memory barrier in
|
|
an atomic instruction. */
|
|
w->protected_tail = saved_protected_tail;
|
|
__cilkrts_fence();
|
|
}
|
|
|
|
/*
|
|
* __cilkrts_disallow_stealing
|
|
*
|
|
* Move the protected_tail to NEW_PROTECTED_TAIL, preventing any
|
|
* frames from being stolen. If NEW_PROTECTED_TAIL is NULL, prevent
|
|
* stealing from the whole queue. The dekker_protocol has been
|
|
* extended to only steal if head+1 is also < protected_tail.
|
|
*/
|
|
|
|
__cilkrts_stack_frame *volatile *__cilkrts_disallow_stealing(
|
|
__cilkrts_worker *w,
|
|
__cilkrts_stack_frame *volatile *new_protected_tail)
|
|
{
|
|
__cilkrts_stack_frame *volatile *saved_protected_tail = w->protected_tail;
|
|
|
|
if (!new_protected_tail)
|
|
new_protected_tail = w->l->ltq;
|
|
|
|
if (w->protected_tail > new_protected_tail) {
|
|
w->protected_tail = new_protected_tail;
|
|
/* Issue a store-store barrier. The update to protected_tail
|
|
here must precede the update to tail in the next spawn.
|
|
On x86 this is probably not needed. */
|
|
#if defined __GNUC__ && __ICC >= 1200 && !(__MIC__ ||__MIC2__)
|
|
_mm_sfence();
|
|
#else
|
|
__cilkrts_fence();
|
|
#endif
|
|
}
|
|
|
|
return saved_protected_tail;
|
|
}
|
|
|
|
/*************************************************************
|
|
Initialization and startup
|
|
*************************************************************/
|
|
|
|
__cilkrts_worker *make_worker(global_state_t *g,
|
|
int self, __cilkrts_worker *w)
|
|
{
|
|
w->self = self;
|
|
w->g = g;
|
|
|
|
w->pedigree.rank = 0; // Initial rank is 0
|
|
w->pedigree.parent = NULL;
|
|
|
|
w->l = (local_state *)__cilkrts_malloc(sizeof(*w->l));
|
|
|
|
__cilkrts_frame_malloc_per_worker_init(w);
|
|
|
|
w->reducer_map = NULL;
|
|
w->current_stack_frame = NULL;
|
|
w->reserved = NULL;
|
|
|
|
w->l->worker_magic_0 = WORKER_MAGIC_0;
|
|
w->l->team = NULL;
|
|
w->l->type = WORKER_FREE;
|
|
|
|
__cilkrts_mutex_init(&w->l->lock);
|
|
__cilkrts_mutex_init(&w->l->steal_lock);
|
|
w->l->do_not_steal = 0;
|
|
w->l->frame_ff = 0;
|
|
w->l->next_frame_ff = 0;
|
|
w->l->last_full_frame = NULL;
|
|
|
|
w->l->ltq = (__cilkrts_stack_frame **)
|
|
__cilkrts_malloc(g->ltqsize * sizeof(*w->l->ltq));
|
|
w->ltq_limit = w->l->ltq + g->ltqsize;
|
|
w->head = w->tail = w->l->ltq;
|
|
|
|
cilk_fiber_pool_init(&w->l->fiber_pool,
|
|
&g->fiber_pool,
|
|
g->stack_size,
|
|
g->fiber_pool_size,
|
|
0, // alloc_max is 0. We don't allocate from the heap directly without checking the parent pool.
|
|
0);
|
|
#if FIBER_DEBUG >= 2
|
|
fprintf(stderr, "ThreadId=%p: Making w=%d (%p), pool = %p\n",
|
|
cilkos_get_current_thread_id(),
|
|
w->self, w,
|
|
&w->l->fiber_pool);
|
|
#endif
|
|
w->l->scheduling_fiber = NULL;
|
|
w->l->original_pedigree_leaf = NULL;
|
|
w->l->rand_seed = 0; /* the scheduler will overwrite this field */
|
|
|
|
w->l->post_suspend = 0;
|
|
w->l->suspended_stack = 0;
|
|
w->l->fiber_to_free = NULL;
|
|
w->l->pending_exception = NULL;
|
|
|
|
#if CILK_PROFILE
|
|
w->l->stats = __cilkrts_malloc(sizeof(statistics));
|
|
__cilkrts_init_stats(w->l->stats);
|
|
#else
|
|
w->l->stats = NULL;
|
|
#endif
|
|
w->l->steal_failure_count = 0;
|
|
|
|
w->l->work_stolen = 0;
|
|
|
|
// Initialize record/replay assuming we're doing neither
|
|
w->l->record_replay_fptr = NULL;
|
|
w->l->replay_list_root = NULL;
|
|
w->l->replay_list_entry = NULL;
|
|
w->l->signal_node = NULL;
|
|
// Nothing's been stolen yet
|
|
w->l->worker_magic_1 = WORKER_MAGIC_1;
|
|
|
|
/*w->parallelism_disabled = 0;*/
|
|
|
|
// Allow stealing all frames. Sets w->saved_protected_tail
|
|
__cilkrts_restore_stealing(w, w->ltq_limit);
|
|
|
|
__cilkrts_init_worker_sysdep(w);
|
|
|
|
reset_THE_exception(w);
|
|
|
|
return w;
|
|
}
|
|
|
|
void destroy_worker(__cilkrts_worker *w)
|
|
{
|
|
CILK_ASSERT (NULL == w->l->pending_exception);
|
|
|
|
// Deallocate the scheduling fiber
|
|
if (NULL != w->l->scheduling_fiber)
|
|
{
|
|
// The scheduling fiber is the main fiber for system workers and must
|
|
// be deallocated by the thread that created it. Thus, we can
|
|
// deallocate only free workers' (formerly user workers) scheduling
|
|
// fibers here.
|
|
CILK_ASSERT(WORKER_FREE == w->l->type);
|
|
|
|
#if FIBER_DEBUG >=1
|
|
fprintf(stderr, "ThreadId=%p, w=%p, %d, deallocating scheduling fiber = %p, \n",
|
|
cilkos_get_current_thread_id(),
|
|
w,
|
|
w->self,
|
|
w->l->scheduling_fiber);
|
|
#endif
|
|
int ref_count = cilk_fiber_remove_reference(w->l->scheduling_fiber, NULL);
|
|
// Scheduling fiber should never have extra references because of exceptions.
|
|
CILK_ASSERT(0 == ref_count);
|
|
w->l->scheduling_fiber = NULL;
|
|
}
|
|
|
|
#if CILK_PROFILE
|
|
if (w->l->stats) {
|
|
__cilkrts_free(w->l->stats);
|
|
}
|
|
#else
|
|
CILK_ASSERT(NULL == w->l->stats);
|
|
#endif
|
|
|
|
/* Free any cached fibers. */
|
|
cilk_fiber_pool_destroy(&w->l->fiber_pool);
|
|
|
|
__cilkrts_destroy_worker_sysdep(w);
|
|
|
|
if (w->l->signal_node) {
|
|
CILK_ASSERT(WORKER_SYSTEM == w->l->type);
|
|
signal_node_destroy(w->l->signal_node);
|
|
}
|
|
|
|
__cilkrts_free(w->l->ltq);
|
|
__cilkrts_mutex_destroy(0, &w->l->lock);
|
|
__cilkrts_mutex_destroy(0, &w->l->steal_lock);
|
|
__cilkrts_frame_malloc_per_worker_cleanup(w);
|
|
|
|
__cilkrts_free(w->l);
|
|
|
|
// The caller is responsible for freeing the worker memory
|
|
}
|
|
|
|
/*
|
|
* Make a worker into a system worker.
|
|
*/
|
|
static void make_worker_system(__cilkrts_worker *w) {
|
|
CILK_ASSERT(WORKER_FREE == w->l->type);
|
|
w->l->type = WORKER_SYSTEM;
|
|
w->l->signal_node = signal_node_create();
|
|
}
|
|
|
|
void __cilkrts_deinit_internal(global_state_t *g)
|
|
{
|
|
int i;
|
|
__cilkrts_worker *w;
|
|
|
|
// If there's no global state then we're done
|
|
if (NULL == g)
|
|
return;
|
|
|
|
#ifdef CILK_PROFILE
|
|
__cilkrts_dump_stats_to_stderr(g);
|
|
#endif
|
|
|
|
w = g->workers[0];
|
|
if (w->l->frame_ff) {
|
|
__cilkrts_destroy_full_frame(w, w->l->frame_ff);
|
|
w->l->frame_ff = 0;
|
|
}
|
|
|
|
// Release any resources used for record/replay
|
|
replay_term(g);
|
|
|
|
// Destroy any system dependent global state
|
|
__cilkrts_destroy_global_sysdep(g);
|
|
|
|
for (i = 0; i < g->total_workers; ++i)
|
|
destroy_worker(g->workers[i]);
|
|
|
|
// Free memory for all worker blocks which were allocated contiguously
|
|
__cilkrts_free(g->workers[0]);
|
|
|
|
__cilkrts_free(g->workers);
|
|
|
|
cilk_fiber_pool_destroy(&g->fiber_pool);
|
|
__cilkrts_frame_malloc_global_cleanup(g);
|
|
|
|
cilkg_deinit_global_state();
|
|
}
|
|
|
|
/*
|
|
* Wake the runtime by notifying the system workers that they can steal. The
|
|
* first user worker into the runtime should call this.
|
|
*/
|
|
static void wake_runtime(global_state_t *g)
|
|
{
|
|
__cilkrts_worker *root;
|
|
if (g->P > 1) {
|
|
// Send a message to the root node. The message will propagate.
|
|
root = g->workers[0];
|
|
CILK_ASSERT(root->l->signal_node);
|
|
signal_node_msg(root->l->signal_node, 1);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Put the runtime to sleep. The last user worker out of the runtime should
|
|
* call this. Like Dad always said, turn out the lights when nobody's in the
|
|
* room.
|
|
*/
|
|
static void sleep_runtime(global_state_t *g)
|
|
{
|
|
__cilkrts_worker *root;
|
|
if (g->P > 1) {
|
|
// Send a message to the root node. The message will propagate.
|
|
root = g->workers[0];
|
|
CILK_ASSERT(root->l->signal_node);
|
|
signal_node_msg(root->l->signal_node, 0);
|
|
}
|
|
}
|
|
|
|
/* Called when a user thread joins Cilk.
|
|
Global lock must be held. */
|
|
void __cilkrts_enter_cilk(global_state_t *g)
|
|
{
|
|
if (g->Q++ == 0) {
|
|
// If this is the first user thread to enter Cilk wake
|
|
// up all the workers.
|
|
wake_runtime(g);
|
|
}
|
|
}
|
|
|
|
/* Called when a user thread leaves Cilk.
|
|
Global lock must be held. */
|
|
void __cilkrts_leave_cilk(global_state_t *g)
|
|
{
|
|
if (--g->Q == 0) {
|
|
// Put the runtime to sleep.
|
|
sleep_runtime(g);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* worker_runnable
|
|
*
|
|
* Return true if the worker should continue to try to steal. False, otherwise.
|
|
*/
|
|
|
|
NOINLINE
|
|
static enum schedule_t worker_runnable(__cilkrts_worker *w)
|
|
{
|
|
global_state_t *g = w->g;
|
|
|
|
/* If this worker has something to do, do it.
|
|
Otherwise the work would be lost. */
|
|
if (w->l->next_frame_ff)
|
|
return SCHEDULE_RUN;
|
|
|
|
// If Cilk has explicitly (by the user) been told to exit (i.e., by
|
|
// __cilkrts_end_cilk() -> __cilkrts_stop_workers(g)), then return 0.
|
|
if (g->work_done)
|
|
return SCHEDULE_EXIT;
|
|
|
|
if (0 == w->self) {
|
|
// This worker is the root node and is the only one that may query the
|
|
// global state to see if there are still any user workers in Cilk.
|
|
if (w->l->steal_failure_count > g->max_steal_failures) {
|
|
if (signal_node_should_wait(w->l->signal_node)) {
|
|
return SCHEDULE_WAIT;
|
|
} else {
|
|
// Reset the steal_failure_count since we have verified that
|
|
// user workers are still in Cilk.
|
|
w->l->steal_failure_count = 0;
|
|
}
|
|
}
|
|
} else if (WORKER_SYSTEM == w->l->type &&
|
|
signal_node_should_wait(w->l->signal_node)) {
|
|
// This worker has been notified by its parent that it should stop
|
|
// trying to steal.
|
|
return SCHEDULE_WAIT;
|
|
}
|
|
|
|
return SCHEDULE_RUN;
|
|
}
|
|
|
|
|
|
|
|
// Initialize the worker structs, but don't start the workers themselves.
|
|
static void init_workers(global_state_t *g)
|
|
{
|
|
int total_workers = g->total_workers;
|
|
int i;
|
|
struct CILK_ALIGNAS(256) buffered_worker {
|
|
__cilkrts_worker w;
|
|
char buf[64];
|
|
} *workers_memory;
|
|
|
|
/* not needed if only one worker */
|
|
cilk_fiber_pool_init(&g->fiber_pool,
|
|
NULL,
|
|
g->stack_size,
|
|
g->global_fiber_pool_size, // buffer_size
|
|
g->max_stacks, // maximum # to allocate
|
|
1);
|
|
|
|
cilk_fiber_pool_set_fiber_limit(&g->fiber_pool,
|
|
(g->max_stacks ? g->max_stacks : INT_MAX));
|
|
|
|
g->workers = (__cilkrts_worker **)
|
|
__cilkrts_malloc(total_workers * sizeof(*g->workers));
|
|
|
|
// Allocate 1 block of memory for workers to make life easier for tools
|
|
// like Inspector which run multithreaded and need to know the memory
|
|
// range for all the workers that will be accessed in a user's program
|
|
workers_memory = (struct buffered_worker*)
|
|
__cilkrts_malloc(sizeof(*workers_memory) * total_workers);
|
|
|
|
// Notify any tools that care (Cilkscreen and Inspector) that they should
|
|
// ignore memory allocated for the workers
|
|
__cilkrts_cilkscreen_ignore_block(&workers_memory[0],
|
|
&workers_memory[total_workers]);
|
|
|
|
// Initialize worker structs, including unused worker slots.
|
|
for (i = 0; i < total_workers; ++i) {
|
|
g->workers[i] = make_worker(g, i, &workers_memory[i].w);
|
|
}
|
|
|
|
// Set the workers in the first P - 1 slots to be system workers.
|
|
// Remaining worker structs already have type == 0.
|
|
for (i = 0; i < g->system_workers; ++i) {
|
|
make_worker_system(g->workers[i]);
|
|
}
|
|
}
|
|
|
|
void __cilkrts_init_internal(int start)
|
|
{
|
|
global_state_t *g = NULL;
|
|
|
|
if (cilkg_is_published()) {
|
|
g = cilkg_init_global_state();
|
|
}
|
|
else {
|
|
|
|
// We think the state has not been published yet.
|
|
// Grab the lock and try to initialize/publish.
|
|
global_os_mutex_lock();
|
|
|
|
if (cilkg_is_published()) {
|
|
// Some other thread must have snuck in and published.
|
|
g = cilkg_init_global_state();
|
|
}
|
|
else {
|
|
// Initialize and retrieve global state
|
|
g = cilkg_init_global_state();
|
|
|
|
// Set the scheduler pointer
|
|
g->scheduler = worker_scheduler_function;
|
|
|
|
// If we're running under a sequential P-Tool (Cilkscreen or
|
|
// Cilkview) then there's only one worker and we need to tell
|
|
// the tool about the extent of the stack
|
|
if (g->under_ptool)
|
|
__cilkrts_establish_c_stack();
|
|
init_workers(g);
|
|
|
|
// Initialize per-work record/replay logging
|
|
replay_init_workers(g);
|
|
|
|
// Initialize any system dependent global state
|
|
__cilkrts_init_global_sysdep(g);
|
|
|
|
|
|
cilkg_publish_global_state(g);
|
|
}
|
|
|
|
global_os_mutex_unlock();
|
|
}
|
|
|
|
CILK_ASSERT(g);
|
|
|
|
if (start && !g->workers_running)
|
|
{
|
|
// Acquire the global OS mutex while we're starting the workers
|
|
global_os_mutex_lock();
|
|
if (!g->workers_running)
|
|
// Start P - 1 system workers since P includes the first user
|
|
// worker.
|
|
__cilkrts_start_workers(g, g->P - 1);
|
|
global_os_mutex_unlock();
|
|
}
|
|
}
|
|
|
|
|
|
/************************************************************************
|
|
Methods for reducer protocol.
|
|
|
|
Reductions occur in two places:
|
|
A. A full frame "ff" is returning from a spawn with a stolen parent.
|
|
B. A full frame "ff" is stalling at a sync.
|
|
|
|
To support parallel reductions, reduction functions need to be
|
|
executed while control is on a user stack, before jumping into the
|
|
runtime. These reductions can not occur while holding a worker or
|
|
frame lock.
|
|
|
|
Before a worker w executes a reduction in either Case A or B, w's
|
|
deque is empty.
|
|
|
|
Since parallel reductions push work onto the deque, we must do extra
|
|
work to set up runtime data structures properly before reductions
|
|
begin to allow stealing. ( Normally, when we have only serial
|
|
reductions, once a worker w starts a reduction, its deque remains
|
|
empty until w either steals another frame or resumes a suspended
|
|
frame. Thus, we don't care about the state of the deque, since w
|
|
will reset its deque when setting up execution of a frame. )
|
|
|
|
To allow for parallel reductions, we coerce the runtime data
|
|
structures so that, from their perspective, it looks as though we
|
|
have spliced in an "execute_reductions()" function. Consider the
|
|
two cases for reductions:
|
|
|
|
Case A: Return from a spawn with a stolen parent.
|
|
Consider a spawned function g is returning on a worker w.
|
|
Assume:
|
|
- g was spawned from a parent function f.
|
|
- ff is the full frame for g's spawn helper
|
|
- sf be the __cilkrts_stack_frame for g's spawn helper.
|
|
|
|
We are conceptually splicing "execute_reductions()" so that it
|
|
occurs immediately before the spawn helper of g returns to f.
|
|
|
|
We do so by creating two different world views --- one for the
|
|
runtime data structures, and one for the actual control flow.
|
|
|
|
- Before reductions begin, the runtime data structures should
|
|
look as though the spawn helper of g is calling
|
|
"execute_reductions()", in terms of both the user stack and
|
|
worker deque. More precisely, w should satisfy the
|
|
following properties:
|
|
|
|
(a) w has ff as its full frame,
|
|
(b) w has sf as its __cilkrts_stack_frame, and
|
|
(c) w has an empty deque.
|
|
|
|
If the runtime satisfies these properties, then if w
|
|
encounters a spawn in a parallel reduction, it can push onto
|
|
a valid deque. Also, when a steal from w occurs, it will
|
|
build the correct tree of full frames when w is stolen from.
|
|
|
|
- In actual control flow, however, once the
|
|
"execute_reductions()" function returns, it is actually
|
|
returning to runtime code instead of g's spawn helper.
|
|
|
|
At the point a worker w began executing reductions, the
|
|
control flow / compiled code had already finished g's spawn
|
|
helper, and w was about to enter the runtime. With parallel
|
|
reductions, some worker v (which might be different from w)
|
|
is the one returning to the runtime.
|
|
|
|
|
|
The reduction logic consists of 4 steps:
|
|
|
|
A1. Restore runtime data structures to make it look as though
|
|
the spawn helper of g() is still the currently executing
|
|
frame for w.
|
|
|
|
A2. Execute reductions on the user stack. Reductions also
|
|
includes the logic for exceptions and stacks. Note that
|
|
reductions start on w, but may finish on a different
|
|
worker if there is parallelism in the reduce.
|
|
|
|
A3. Splice out ff from the tree of full frames.
|
|
|
|
A4. Jump into the runtime/scheduling stack and execute
|
|
"do_return_from_spawn". This method
|
|
|
|
(a) Frees the user stack we were just on if it is no longer needed.
|
|
(b) Decrement the join counter on ff->parent, and tries to do a
|
|
provably good steal.
|
|
(c) Clean up the full frame ff.
|
|
|
|
|
|
Case B: Stalling at a sync.
|
|
|
|
Consider a function g(), with full frame ff and
|
|
__cilkrts_stack_frame sf. Suppose g() stalls at a sync, and we
|
|
are executing reductions.
|
|
|
|
Conceptually, we are splicing in an "execute_reductions()"
|
|
function into g() as the last action that g() takes immediately
|
|
before it executes the cilk_sync.
|
|
|
|
The reduction logic for this case is similar to Case A.
|
|
|
|
B1. Restore the runtime data structures.
|
|
|
|
The main difference from Case A is that ff/sf is still a
|
|
frame that needs to be executed later (since it is stalling
|
|
at a cilk_sync). Thus, we also need to save the current
|
|
stack information into "ff" so that we can correctly resume
|
|
execution of "ff" after the sync.
|
|
|
|
B2. Execute reductions on the user stack.
|
|
|
|
B3. No frame to splice out of the tree.
|
|
|
|
B4. Jump into the runtime/scheduling stack and execute "do_sync".
|
|
This method:
|
|
(a) Frees the user stack we were just on if it is no longer needed.
|
|
(b) Tries to execute a provably good steal.
|
|
|
|
Finally, for the reducer protocol, we consider two reduction paths,
|
|
namely a "fast" and "slow" path. On a fast path, only trivial
|
|
merges of reducer maps happen (i.e., one or both of the maps are
|
|
NULL). Otherwise, on the slow path, a reduction actually needs to
|
|
happen.
|
|
|
|
*****************************************************************/
|
|
|
|
/**
|
|
* @brief Locations to store the result of a reduction.
|
|
*
|
|
* Struct storing pointers to the fields in our "left" sibling that we
|
|
* should update when splicing out a full frame or stalling at a sync.
|
|
*/
|
|
typedef struct {
|
|
/** A pointer to the location of our left reducer map. */
|
|
struct cilkred_map **map_ptr;
|
|
|
|
/** A pointer to the location of our left exception. */
|
|
struct pending_exception_info **exception_ptr;
|
|
} splice_left_ptrs;
|
|
|
|
/**
|
|
* For a full frame returning from a spawn, calculate the pointers to
|
|
* the maps and exceptions to my left.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff Full frame that is dying
|
|
* @return Pointers to our "left" for reducers and exceptions.
|
|
*/
|
|
static inline
|
|
splice_left_ptrs compute_left_ptrs_for_spawn_return(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we hold the lock on ff->parent
|
|
|
|
splice_left_ptrs left_ptrs;
|
|
if (ff->left_sibling) {
|
|
left_ptrs.map_ptr = &ff->left_sibling->right_reducer_map;
|
|
left_ptrs.exception_ptr = &ff->left_sibling->right_pending_exception;
|
|
}
|
|
else {
|
|
full_frame *parent_ff = ff->parent;
|
|
left_ptrs.map_ptr = &parent_ff->children_reducer_map;
|
|
left_ptrs.exception_ptr = &parent_ff->child_pending_exception;
|
|
}
|
|
return left_ptrs;
|
|
}
|
|
|
|
/**
|
|
* For a full frame at a sync, calculate the pointers to the maps and
|
|
* exceptions to my left.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff Full frame that is stalling at a sync.
|
|
* @return Pointers to our "left" for reducers and exceptions.
|
|
*/
|
|
static inline
|
|
splice_left_ptrs compute_left_ptrs_for_sync(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we hold the lock on ff
|
|
splice_left_ptrs left_ptrs;
|
|
|
|
// Figure out which map to the left we should merge into.
|
|
if (ff->rightmost_child) {
|
|
CILK_ASSERT(ff->rightmost_child->parent == ff);
|
|
left_ptrs.map_ptr = &(ff->rightmost_child->right_reducer_map);
|
|
left_ptrs.exception_ptr = &(ff->rightmost_child->right_pending_exception);
|
|
}
|
|
else {
|
|
// We have no children. Then, we should be the last
|
|
// worker at the sync... "left" is our child map.
|
|
left_ptrs.map_ptr = &(ff->children_reducer_map);
|
|
left_ptrs.exception_ptr = &(ff->child_pending_exception);
|
|
}
|
|
return left_ptrs;
|
|
}
|
|
|
|
/**
|
|
* After we have completed all reductions on a spawn return, call this
|
|
* method to finish up before jumping into the runtime.
|
|
*
|
|
* 1. Perform the "reduction" on stacks, i.e., execute the left
|
|
* holder logic to pass the leftmost stack up.
|
|
*
|
|
* w->l->fiber_to_free holds any stack that needs to be freed
|
|
* when control switches into the runtime fiber.
|
|
*
|
|
* 2. Unlink and remove child_ff from the tree of full frames.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param parent_ff The parent of child_ff.
|
|
* @param child_ff The full frame returning from a spawn.
|
|
*/
|
|
static inline
|
|
void finish_spawn_return_on_user_stack(__cilkrts_worker *w,
|
|
full_frame *parent_ff,
|
|
full_frame *child_ff)
|
|
{
|
|
CILK_ASSERT(w->l->fiber_to_free == NULL);
|
|
|
|
// Execute left-holder logic for stacks.
|
|
if (child_ff->left_sibling || parent_ff->fiber_child) {
|
|
// Case where we are not the leftmost stack.
|
|
CILK_ASSERT(parent_ff->fiber_child != child_ff->fiber_self);
|
|
|
|
// Remember any fiber we need to free in the worker.
|
|
// After we jump into the runtime, we will actually do the
|
|
// free.
|
|
w->l->fiber_to_free = child_ff->fiber_self;
|
|
}
|
|
else {
|
|
// We are leftmost, pass stack/fiber up to parent.
|
|
// Thus, no stack/fiber to free.
|
|
parent_ff->fiber_child = child_ff->fiber_self;
|
|
w->l->fiber_to_free = NULL;
|
|
}
|
|
|
|
child_ff->fiber_self = NULL;
|
|
|
|
unlink_child(parent_ff, child_ff);
|
|
}
|
|
|
|
|
|
/**
|
|
* Executes any fast reductions necessary to splice ff out of the tree
|
|
* of full frames.
|
|
*
|
|
* This "fast" path performs only trivial merges of reducer maps,
|
|
* i.e,. when one of them is NULL.
|
|
* (See slow_path_reductions_for_spawn_return() for slow path.)
|
|
*
|
|
* Returns: 1 if we finished all reductions.
|
|
* Returns: 0 if there are still reductions to execute, and
|
|
* we should execute the slow path.
|
|
*
|
|
* This method assumes w holds the frame lock on parent_ff.
|
|
* After this method completes:
|
|
* 1. We have spliced ff out of the tree of full frames.
|
|
* 2. The reducer maps of child_ff have been deposited
|
|
* "left" according to the reducer protocol.
|
|
* 3. w->l->stack_to_free stores the stack
|
|
* that needs to be freed once we jump into the runtime.
|
|
*
|
|
* We have not, however, decremented the join counter on ff->parent.
|
|
* This prevents any other workers from resuming execution of the parent.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame returning from a spawn.
|
|
* @return NULL if we finished all reductions.
|
|
* @return The address where the left map is stored (which should be passed to
|
|
* slow_path_reductions_for_spawn_return()) if there are
|
|
* still reductions to execute.
|
|
*/
|
|
struct cilkred_map**
|
|
fast_path_reductions_for_spawn_return(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
// ASSERT: we hold ff->parent->lock.
|
|
splice_left_ptrs left_ptrs;
|
|
|
|
CILK_ASSERT(NULL == w->l->pending_exception);
|
|
|
|
// Figure out the pointers to the left where I want
|
|
// to put reducers and exceptions.
|
|
left_ptrs = compute_left_ptrs_for_spawn_return(w, ff);
|
|
|
|
// Go ahead and merge exceptions while holding the lock.
|
|
splice_exceptions_for_spawn(w, ff, left_ptrs.exception_ptr);
|
|
|
|
// Now check if we have any reductions to perform.
|
|
//
|
|
// Consider all the cases of left, middle and right maps.
|
|
// 0. (-, -, -) : finish and return 1
|
|
// 1. (L, -, -) : finish and return 1
|
|
// 2. (-, M, -) : slide over to left, finish, and return 1.
|
|
// 3. (L, M, -) : return 0
|
|
// 4. (-, -, R) : slide over to left, finish, and return 1.
|
|
// 5. (L, -, R) : return 0
|
|
// 6. (-, M, R) : return 0
|
|
// 7. (L, M, R) : return 0
|
|
//
|
|
// In terms of code:
|
|
// L == *left_ptrs.map_ptr
|
|
// M == w->reducer_map
|
|
// R == f->right_reducer_map.
|
|
//
|
|
// The goal of the code below is to execute the fast path with
|
|
// as few branches and writes as possible.
|
|
|
|
int case_value = (*(left_ptrs.map_ptr) != NULL);
|
|
case_value += ((w->reducer_map != NULL) << 1);
|
|
case_value += ((ff->right_reducer_map != NULL) << 2);
|
|
|
|
// Fastest path is case_value == 0 or 1.
|
|
if (case_value >=2) {
|
|
switch (case_value) {
|
|
case 2:
|
|
*(left_ptrs.map_ptr) = w->reducer_map;
|
|
w->reducer_map = NULL;
|
|
return NULL;
|
|
break;
|
|
case 4:
|
|
*(left_ptrs.map_ptr) = ff->right_reducer_map;
|
|
ff->right_reducer_map = NULL;
|
|
return NULL;
|
|
default:
|
|
// If we have to execute the slow path, then
|
|
// return the pointer to the place to deposit the left
|
|
// map.
|
|
return left_ptrs.map_ptr;
|
|
}
|
|
}
|
|
|
|
// Do nothing
|
|
return NULL;
|
|
}
|
|
|
|
|
|
/**
|
|
* Executes any reductions necessary to splice "ff" frame out of
|
|
* the steal tree.
|
|
*
|
|
* This method executes the "slow" path for reductions on a spawn
|
|
* return, i.e., there are non-NULL maps that need to be merged
|
|
* together.
|
|
*
|
|
* This method should execute only if
|
|
* fast_path_reductions_for_spawn_return() returns a non-NULL
|
|
* left_map_ptr.
|
|
*
|
|
* Upon entry, left_map_ptr should be the location of the left map
|
|
* at the start of the reduction, as calculated by
|
|
* fast_path_reductions_for_spawn_return().
|
|
*
|
|
* After this method completes:
|
|
* 1. We have spliced ff out of the tree of full frames.
|
|
* 2. The reducer maps of child_ff have been deposited
|
|
* "left" according to the reducer protocol.
|
|
* 3. w->l->stack_to_free stores the stack
|
|
* that needs to be freed once we jump into the runtime.
|
|
* We have not, however, decremented the join counter on ff->parent,
|
|
* so no one can resume execution of the parent yet.
|
|
*
|
|
* WARNING:
|
|
* This method assumes the lock on ff->parent is held upon entry, and
|
|
* Upon exit, the worker that returns still holds a lock on ff->parent
|
|
* This method can, however, release and reacquire the lock on ff->parent.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame returning from a spawn.
|
|
* @param left_map_ptr Pointer to our initial left map.
|
|
* @return The worker that this method returns on.
|
|
*/
|
|
static __cilkrts_worker*
|
|
slow_path_reductions_for_spawn_return(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
struct cilkred_map **left_map_ptr)
|
|
{
|
|
|
|
// CILK_ASSERT: w is holding frame lock on parent_ff.
|
|
#if REDPAR_DEBUG > 0
|
|
CILK_ASSERT(!ff->rightmost_child);
|
|
CILK_ASSERT(!ff->is_call_child);
|
|
#endif
|
|
|
|
// Loop invariant:
|
|
// When beginning this loop, we should
|
|
// 1. Be holding the lock on ff->parent.
|
|
// 2. left_map_ptr should be the address of the pointer to the left map.
|
|
// 3. All maps should be slid over left by one, if possible.
|
|
// 4. All exceptions should be merged so far.
|
|
while (1) {
|
|
|
|
// Slide middle map left if possible.
|
|
if (!(*left_map_ptr)) {
|
|
*left_map_ptr = w->reducer_map;
|
|
w->reducer_map = NULL;
|
|
}
|
|
// Slide right map to middle if possible.
|
|
if (!w->reducer_map) {
|
|
w->reducer_map = ff->right_reducer_map;
|
|
ff->right_reducer_map = NULL;
|
|
}
|
|
|
|
// Since we slid everything left by one,
|
|
// we are finished if there is no middle map.
|
|
if (!w->reducer_map) {
|
|
verify_current_wkr(w);
|
|
return w;
|
|
}
|
|
else {
|
|
struct cilkred_map* left_map;
|
|
struct cilkred_map* middle_map;
|
|
struct cilkred_map* right_map;
|
|
|
|
// Take all the maps from their respective locations.
|
|
// We can't leave them in place and execute a reduction because these fields
|
|
// might change once we release the lock.
|
|
left_map = *left_map_ptr;
|
|
*left_map_ptr = NULL;
|
|
middle_map = w->reducer_map;
|
|
w->reducer_map = NULL;
|
|
right_map = ff->right_reducer_map;
|
|
ff->right_reducer_map = NULL;
|
|
|
|
// WARNING!!! Lock release here.
|
|
// We have reductions to execute (and we can't hold locks).
|
|
__cilkrts_frame_unlock(w, ff->parent);
|
|
|
|
// Merge all reducers into the left map.
|
|
left_map = repeated_merge_reducer_maps(&w,
|
|
left_map,
|
|
middle_map);
|
|
verify_current_wkr(w);
|
|
left_map = repeated_merge_reducer_maps(&w,
|
|
left_map,
|
|
right_map);
|
|
verify_current_wkr(w);
|
|
CILK_ASSERT(NULL == w->reducer_map);
|
|
// Put the final answer back into w->reducer_map.
|
|
w->reducer_map = left_map;
|
|
|
|
// Save any exceptions generated because of the reduction
|
|
// process from the returning worker. These get merged
|
|
// the next time around the loop.
|
|
CILK_ASSERT(NULL == ff->pending_exception);
|
|
ff->pending_exception = w->l->pending_exception;
|
|
w->l->pending_exception = NULL;
|
|
|
|
// Lock ff->parent for the next loop around.
|
|
__cilkrts_frame_lock(w, ff->parent);
|
|
|
|
// Once we have the lock again, recompute who is to our
|
|
// left.
|
|
splice_left_ptrs left_ptrs;
|
|
left_ptrs = compute_left_ptrs_for_spawn_return(w, ff);
|
|
|
|
// Update the pointer for the left map.
|
|
left_map_ptr = left_ptrs.map_ptr;
|
|
// Splice the exceptions for spawn.
|
|
splice_exceptions_for_spawn(w, ff, left_ptrs.exception_ptr);
|
|
}
|
|
}
|
|
// We should never break out of this loop.
|
|
|
|
CILK_ASSERT(0);
|
|
return NULL;
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* Execute reductions when returning from a spawn whose parent has
|
|
* been stolen.
|
|
*
|
|
* Execution may start on w, but may finish on a different worker.
|
|
* This method acquires/releases the lock on ff->parent.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame of the spawned function that is returning.
|
|
* @param returning_sf The __cilkrts_stack_frame for this returning function.
|
|
* @return The worker returning from this method.
|
|
*/
|
|
static __cilkrts_worker*
|
|
execute_reductions_for_spawn_return(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *returning_sf)
|
|
{
|
|
// Step A1 from reducer protocol described above.
|
|
//
|
|
// Coerce the runtime into thinking that
|
|
// ff/returning_sf are still on the bottom of
|
|
// w's deque.
|
|
restore_frame_for_spawn_return_reduction(w, ff, returning_sf);
|
|
|
|
// Step A2 and A3: Execute reductions on user stack.
|
|
BEGIN_WITH_FRAME_LOCK(w, ff->parent) {
|
|
struct cilkred_map **left_map_ptr;
|
|
left_map_ptr = fast_path_reductions_for_spawn_return(w, ff);
|
|
|
|
// Pointer will be non-NULL if there are
|
|
// still reductions to execute.
|
|
if (left_map_ptr) {
|
|
// WARNING: This method call may release the lock
|
|
// on ff->parent and re-acquire it (possibly on a
|
|
// different worker).
|
|
// We can't hold locks while actually executing
|
|
// reduce functions.
|
|
w = slow_path_reductions_for_spawn_return(w,
|
|
ff,
|
|
left_map_ptr);
|
|
verify_current_wkr(w);
|
|
}
|
|
|
|
finish_spawn_return_on_user_stack(w, ff->parent, ff);
|
|
// WARNING: the use of this lock macro is deceptive.
|
|
// The worker may have changed here.
|
|
} END_WITH_FRAME_LOCK(w, ff->parent);
|
|
return w;
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
* Execute fast "reductions" when ff stalls at a sync.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame stalling at a sync.
|
|
* @return 1 if we are finished with all reductions after calling this method.
|
|
* @return 0 if we still need to execute the slow path reductions.
|
|
*/
|
|
static inline
|
|
int fast_path_reductions_for_sync(__cilkrts_worker *w,
|
|
full_frame *ff) {
|
|
// Return 0 if there is some reduction that needs to happen.
|
|
return !(w->reducer_map || ff->pending_exception);
|
|
}
|
|
|
|
/**
|
|
* Executes slow reductions when ff stalls at a sync.
|
|
* This method should execute only if
|
|
* fast_path_reductions_for_sync(w, ff) returned 0.
|
|
*
|
|
* After this method completes:
|
|
* 1. ff's current reducer map has been deposited into
|
|
* right_reducer_map of ff's rightmost child, or
|
|
* ff->children_reducer_map if ff has no children.
|
|
* 2. Similarly for ff's current exception.
|
|
* 3. Nothing to calculate for stacks --- if we are stalling
|
|
* we will always free a stack.
|
|
*
|
|
* This method may repeatedly acquire/release the lock on ff.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame stalling at a sync.
|
|
* @return The worker returning from this method.
|
|
*/
|
|
static __cilkrts_worker*
|
|
slow_path_reductions_for_sync(__cilkrts_worker *w,
|
|
full_frame *ff)
|
|
{
|
|
struct cilkred_map *left_map;
|
|
struct cilkred_map *middle_map;
|
|
|
|
#if (REDPAR_DEBUG > 0)
|
|
CILK_ASSERT(ff);
|
|
CILK_ASSERT(w->head == w->tail);
|
|
#endif
|
|
|
|
middle_map = w->reducer_map;
|
|
w->reducer_map = NULL;
|
|
|
|
// Loop invariant: middle_map should be valid (the current map to reduce).
|
|
// left_map is junk.
|
|
// w->reducer_map == NULL.
|
|
while (1) {
|
|
BEGIN_WITH_FRAME_LOCK(w, ff) {
|
|
splice_left_ptrs left_ptrs = compute_left_ptrs_for_sync(w, ff);
|
|
|
|
// Grab the "left" map and store pointers to those locations.
|
|
left_map = *(left_ptrs.map_ptr);
|
|
*(left_ptrs.map_ptr) = NULL;
|
|
|
|
// Slide the maps in our struct left as far as possible.
|
|
if (!left_map) {
|
|
left_map = middle_map;
|
|
middle_map = NULL;
|
|
}
|
|
|
|
*(left_ptrs.exception_ptr) =
|
|
__cilkrts_merge_pending_exceptions(w,
|
|
*left_ptrs.exception_ptr,
|
|
ff->pending_exception);
|
|
ff->pending_exception = NULL;
|
|
|
|
// If there is no middle map, then we are done.
|
|
// Deposit left and return.
|
|
if (!middle_map) {
|
|
*(left_ptrs).map_ptr = left_map;
|
|
#if (REDPAR_DEBUG > 0)
|
|
CILK_ASSERT(NULL == w->reducer_map);
|
|
#endif
|
|
// Sanity check upon leaving the loop.
|
|
verify_current_wkr(w);
|
|
// Make sure to unlock before we return!
|
|
__cilkrts_frame_unlock(w, ff);
|
|
return w;
|
|
}
|
|
} END_WITH_FRAME_LOCK(w, ff);
|
|
|
|
// If we get here, we have a nontrivial reduction to execute.
|
|
middle_map = repeated_merge_reducer_maps(&w,
|
|
left_map,
|
|
middle_map);
|
|
verify_current_wkr(w);
|
|
|
|
// Save any exceptions generated because of the reduction
|
|
// process. These get merged the next time around the
|
|
// loop.
|
|
CILK_ASSERT(NULL == ff->pending_exception);
|
|
ff->pending_exception = w->l->pending_exception;
|
|
w->l->pending_exception = NULL;
|
|
}
|
|
|
|
// We should never break out of the loop above.
|
|
CILK_ASSERT(0);
|
|
return NULL;
|
|
}
|
|
|
|
|
|
/**
|
|
* Execute reductions when ff stalls at a sync.
|
|
*
|
|
* Execution starts on w, but may finish on a different worker.
|
|
* This method may acquire/release the lock on ff.
|
|
*
|
|
* @param w The currently executing worker.
|
|
* @param ff The full frame of the spawned function at the sync
|
|
* @param sf_at_sync The __cilkrts_stack_frame stalling at a sync
|
|
* @return The worker returning from this method.
|
|
*/
|
|
static __cilkrts_worker*
|
|
execute_reductions_for_sync(__cilkrts_worker *w,
|
|
full_frame *ff,
|
|
__cilkrts_stack_frame *sf_at_sync)
|
|
{
|
|
int finished_reductions;
|
|
// Step B1 from reducer protocol above:
|
|
// Restore runtime invariants.
|
|
//
|
|
// The following code for this step is almost equivalent to
|
|
// the following sequence:
|
|
// 1. disown(w, ff, sf_at_sync, "sync") (which itself
|
|
// calls make_unrunnable(w, ff, sf_at_sync))
|
|
// 2. make_runnable(w, ff, sf_at_sync).
|
|
//
|
|
// The "disown" will mark the frame "sf_at_sync"
|
|
// as stolen and suspended, and save its place on the stack,
|
|
// so it can be resumed after the sync.
|
|
//
|
|
// The difference is, that we don't want the disown to
|
|
// break the following connections yet, since we are
|
|
// about to immediately make sf/ff runnable again anyway.
|
|
// sf_at_sync->worker == w
|
|
// w->l->frame_ff == ff.
|
|
//
|
|
// These connections are needed for parallel reductions, since
|
|
// we will use sf / ff as the stack frame / full frame for
|
|
// executing any potential reductions.
|
|
//
|
|
// TBD: Can we refactor the disown / make_unrunnable code
|
|
// to avoid the code duplication here?
|
|
|
|
ff->call_stack = NULL;
|
|
|
|
// Normally, "make_unrunnable" would add CILK_FRAME_STOLEN and
|
|
// CILK_FRAME_SUSPENDED to sf_at_sync->flags and save the state of
|
|
// the stack so that a worker can resume the frame in the correct
|
|
// place.
|
|
//
|
|
// But on this path, CILK_FRAME_STOLEN should already be set.
|
|
// Also, we technically don't want to suspend the frame until
|
|
// the reduction finishes.
|
|
// We do, however, need to save the stack before
|
|
// we start any reductions, since the reductions might push more
|
|
// data onto the stack.
|
|
CILK_ASSERT(sf_at_sync->flags | CILK_FRAME_STOLEN);
|
|
|
|
__cilkrts_put_stack(ff, sf_at_sync);
|
|
__cilkrts_make_unrunnable_sysdep(w, ff, sf_at_sync, 1,
|
|
"execute_reductions_for_sync");
|
|
CILK_ASSERT(w->l->frame_ff == ff);
|
|
|
|
// Step B2: Execute reductions on user stack.
|
|
// Check if we have any "real" reductions to do.
|
|
finished_reductions = fast_path_reductions_for_sync(w, ff);
|
|
|
|
if (!finished_reductions) {
|
|
// Still have some real reductions to execute.
|
|
// Run them here.
|
|
|
|
// This method may acquire/release the lock on ff.
|
|
w = slow_path_reductions_for_sync(w, ff);
|
|
|
|
// The previous call may return on a different worker.
|
|
// than what we started on.
|
|
verify_current_wkr(w);
|
|
}
|
|
|
|
#if REDPAR_DEBUG >= 0
|
|
CILK_ASSERT(w->l->frame_ff == ff);
|
|
CILK_ASSERT(ff->call_stack == NULL);
|
|
#endif
|
|
|
|
// Now we suspend the frame ff (since we've
|
|
// finished the reductions). Roughly, we've split apart the
|
|
// "make_unrunnable" call here --- we've already saved the
|
|
// stack info earlier before the reductions execute.
|
|
// All that remains is to restore the call stack back into the
|
|
// full frame, and mark the frame as suspended.
|
|
ff->call_stack = sf_at_sync;
|
|
sf_at_sync->flags |= CILK_FRAME_SUSPENDED;
|
|
|
|
// At a nontrivial sync, we should always free the current fiber,
|
|
// because it can not be leftmost.
|
|
w->l->fiber_to_free = ff->fiber_self;
|
|
ff->fiber_self = NULL;
|
|
return w;
|
|
}
|
|
|
|
|
|
/*
|
|
Local Variables: **
|
|
c-file-style:"bsd" **
|
|
c-basic-offset:4 **
|
|
indent-tabs-mode:nil **
|
|
End: **
|
|
*/
|