AllocateSchedulerPtr = itt_present ? &custom_scheduler<DefaultSchedulerTraits>::allocate_scheduler :

&custom_scheduler<IntelSchedulerTraits>::allocate_scheduler;

#if __TBB_TASK_GROUP_CONTEXT

// There must be no tasks belonging to this fake task group. Mark invalid for the assert

__TBB_ASSERT(!(task_group_context::low_unused_state_bit & (task_group_context::low_unused_state_bit-1)), NULL);

the_dummy_context.my_state = task_group_context::low_unused_state_bit;

#if __TBB_TASK_PRIORITY

// It should never prevent tasks from being passed to execution.

the_dummy_context.my_priority = num_priority_levels - 1;

#endif /* __TBB_TASK_PRIORITY */

#endif /* __TBB_TASK_GROUP_CONTEXT */

: my_stealing_threshold(0)

, my_market(NULL)

, my_random( this )

, my_free_list(NULL)

#if __TBB_HOARD_NONLOCAL_TASKS

, my_nonlocal_free_list(NULL)

#endif

, my_dummy_task(NULL)

, my_ref_count(1)

, my_auto_initialized(false)

#if __TBB_COUNT_TASK_NODES

, my_task_node_count(0)

#endif /* __TBB_COUNT_TASK_NODES */

, my_small_task_count(1) // Extra 1 is a guard reference

, my_return_list(NULL)

#if __TBB_TASK_GROUP_CONTEXT

, my_local_ctx_list_update(make_atomic(uintptr_t(0)))

#endif /* __TBB_TASK_GROUP_CONTEXT */

#if __TBB_TASK_PRIORITY

, my_offloaded_tasks(NULL)

, my_offloaded_task_list_tail_link(NULL)

, my_local_reload_epoch(0)

, my_pool_reshuffling_pending(false)

#endif /* __TBB_TASK_PRIORITY */

#if __TBB_TASK_GROUP_CONTEXT

, my_nonlocal_ctx_list_update(make_atomic(uintptr_t(0)))

#endif /* __TBB_TASK_GROUP_CONTEXT */

#if __TBB_SURVIVE_THREAD_SWITCH && TBB_USE_ASSERT

, my_cilk_state(cs_none)

#endif /* __TBB_SURVIVE_THREAD_SWITCH && TBB_USE_ASSERT */

{

my_arena_index = index;

my_arena_slot = 0;

my_arena = a;

my_innermost_running_task = NULL;

my_dispatching_task = NULL;

my_affinity_id = 0;

#if __TBB_SCHEDULER_OBSERVER

my_last_global_observer = NULL;

my_last_local_observer = NULL;

#endif /* __TBB_SCHEDULER_OBSERVER */

#if __TBB_TASK_PRIORITY

my_ref_top_priority = NULL;

my_ref_reload_epoch = NULL;

#endif /* __TBB_TASK_PRIORITY */

my_dummy_task = &allocate_task( sizeof(task), __TBB_CONTEXT_ARG(NULL, NULL) );

#if __TBB_TASK_GROUP_CONTEXT

my_context_list_head.my_prev = &my_context_list_head;

my_context_list_head.my_next = &my_context_list_head;

ITT_SYNC_CREATE(&my_context_list_mutex, SyncType_Scheduler, SyncObj_ContextsList);

#endif /* __TBB_TASK_GROUP_CONTEXT */

my_dummy_task->prefix().ref_count = 2;

ITT_SYNC_CREATE(&my_dummy_task->prefix().ref_count, SyncType_Scheduler, SyncObj_WorkerLifeCycleMgmt);

ITT_SYNC_CREATE(&my_return_list, SyncType_Scheduler, SyncObj_TaskReturnList);

assert_task_pool_valid();

#if __TBB_SURVIVE_THREAD_SWITCH

my_cilk_unwatch_thunk.routine = NULL;

#endif /* __TBB_SURVIVE_THREAD_SWITCH */

acquire_task_pool();

task** tp = my_arena_slot->task_pool_ptr;

__TBB_ASSERT( my_arena_slot->my_task_pool_size >= min_task_pool_size, NULL );

const size_t H = __TBB_load_relaxed(my_arena_slot->head); // mirror

const size_t T = __TBB_load_relaxed(my_arena_slot->tail); // mirror

__TBB_ASSERT( H <= T, NULL );

for ( size_t i = 0; i < H; ++i )

__TBB_ASSERT( tp[i] == poisoned_ptr, "Task pool corrupted" );

for ( size_t i = H; i < T; ++i ) {

__TBB_ASSERT( (uintptr_t)tp[i] + 1 > 1u, "nil or invalid task pointer in the deque" );

__TBB_ASSERT( tp[i]->prefix().state == task::ready ||

tp[i]->prefix().extra_state == es_task_proxy, "task in the deque has invalid state" );

}

for ( size_t i = T; i < my_arena_slot->my_task_pool_size; ++i )

__TBB_ASSERT( tp[i] == poisoned_ptr, "Task pool corrupted" );

release_task_pool();

// Stacks are growing top-down. Highest address is called "stack base",

// and the lowest is "stack limit".

__TBB_ASSERT( !my_stealing_threshold, "Stealing threshold has already been calculated" );

size_t stack_size = my_market->worker_stack_size();

#if USE_WINTHREAD

#if defined(_MSC_VER)&&_MSC_VER<1400 && !_WIN64

NT_TIB *pteb = (NT_TIB*)__TBB_machine_get_current_teb();

#else

NT_TIB *pteb = (NT_TIB*)NtCurrentTeb();

#endif

__TBB_ASSERT( &pteb < pteb->StackBase && &pteb > pteb->StackLimit, "invalid stack info in TEB" );

__TBB_ASSERT( stack_size >0, "stack_size not initialized?" );

// When a thread is created with the attribute STACK_SIZE_PARAM_IS_A_RESERVATION, stack limit

// in the TIB points to the committed part of the stack only. This renders the expression

// "(uintptr_t)pteb->StackBase / 2 + (uintptr_t)pteb->StackLimit / 2" virtually useless.

// Thus for worker threads we use the explicit stack size we used while creating them.

// And for master threads we rely on the following fact and assumption:

// - the default stack size of a master thread on Windows is 1M;

// - if it was explicitly set by the application it is at least as large as the size of a worker stack.

if ( is_worker() || stack_size < MByte )

my_stealing_threshold = (uintptr_t)pteb->StackBase - stack_size / 2;

else

my_stealing_threshold = (uintptr_t)pteb->StackBase - MByte / 2;

#else /* USE_PTHREAD */

// There is no portable way to get stack base address in Posix, so we use

// non-portable method (on all modern Linux) or the simplified approach

// based on the common sense assumptions. The most important assumption

// is that the main thread's stack size is not less than that of other threads.

// See also comment 3 at the end of this file

void *stack_base = &stack_size;

#if __linux__ && !__bg__

#if __TBB_ipf

void *rsb_base = __TBB_get_bsp();

#endif

size_t np_stack_size = 0;

void *stack_limit = NULL;

pthread_attr_t np_attr_stack;

if( 0 == pthread_getattr_np(pthread_self(), &np_attr_stack) ) {

if ( 0 == pthread_attr_getstack(&np_attr_stack, &stack_limit, &np_stack_size) ) {

#if __TBB_ipf

pthread_attr_t attr_stack;

if ( 0 == pthread_attr_init(&attr_stack) ) {

if ( 0 == pthread_attr_getstacksize(&attr_stack, &stack_size) ) {

if ( np_stack_size < stack_size ) {

// We are in a secondary thread. Use reliable data.

// IA-64 architecture stack is split into RSE backup and memory parts

rsb_base = stack_limit;

stack_size = np_stack_size/2;

// Limit of the memory part of the stack

stack_limit = (char*)stack_limit + stack_size;

}

// We are either in the main thread or this thread stack

// is bigger that that of the main one. As we cannot discern

// these cases we fall back to the default (heuristic) values.

}

pthread_attr_destroy(&attr_stack);

}

// IA-64 architecture stack is split into RSE backup and memory parts

my_rsb_stealing_threshold = (uintptr_t)((char*)rsb_base + stack_size/2);

#endif /* __TBB_ipf */

// Size of the stack free part

stack_size = size_t((char*)stack_base - (char*)stack_limit);

}

pthread_attr_destroy(&np_attr_stack);

}

#endif /* __linux__ */

__TBB_ASSERT( stack_size>0, "stack size must be positive" );

my_stealing_threshold = (uintptr_t)((char*)stack_base - stack_size/2);

#endif /* USE_PTHREAD */

// Detach contexts remaining in the local list

bool wait_for_concurrent_destroyers_to_leave = false;

uintptr_t local_count_snapshot = my_context_state_propagation_epoch;

my_local_ctx_list_update.store<relaxed>(1);

{

// This is just a definition. Actual lock is acquired only in case of conflict.

spin_mutex::scoped_lock lock;

// Full fence prevents reordering of store to my_local_ctx_list_update with

// load from my_nonlocal_ctx_list_update.

atomic_fence();

// Check for the conflict with concurrent destroyer or cancellation propagator

if ( my_nonlocal_ctx_list_update.load<relaxed>() || local_count_snapshot != the_context_state_propagation_epoch )

lock.acquire(my_context_list_mutex);

// No acquire fence is necessary for loading my_context_list_head.my_next,

// as the list can be updated by this thread only.

context_list_node_t *node = my_context_list_head.my_next;

while ( node != &my_context_list_head ) {

task_group_context &ctx = __TBB_get_object_ref(task_group_context, my_node, node);

__TBB_ASSERT( __TBB_load_relaxed(ctx.my_kind) != task_group_context::binding_required, "Only a context bound to a root task can be detached" );

node = node->my_next;

__TBB_ASSERT( is_alive(ctx.my_version_and_traits), "Walked into a destroyed context while detaching contexts from the local list" );

// Synchronizes with ~task_group_context(). TODO: evaluate and perhaps relax

if ( internal::as_atomic(ctx.my_kind).fetch_and_store(task_group_context::detached) == task_group_context::dying )

wait_for_concurrent_destroyers_to_leave = true;

}

}

my_local_ctx_list_update.store<release>(0);

// Wait until other threads referencing this scheduler object finish with it

if ( wait_for_concurrent_destroyers_to_leave )

spin_wait_until_eq( my_nonlocal_ctx_list_update, 0u );

__TBB_ASSERT( !my_arena_slot, NULL );

#if __TBB_TASK_GROUP_CONTEXT

cleanup_local_context_list();

#endif /* __TBB_TASK_GROUP_CONTEXT */

free_task<small_local_task>( *my_dummy_task );

#if __TBB_HOARD_NONLOCAL_TASKS

while( task* t = my_nonlocal_free_list ) {

task_prefix& p = t->prefix();

my_nonlocal_free_list = p.next;

__TBB_ASSERT( p.origin && p.origin!=this, NULL );

free_nonlocal_small_task(*t);

}

#endif

// k accounts for a guard reference and each task that we deallocate.

intptr_t k = 1;

for(;;) {

while( task* t = my_free_list ) {

my_free_list = t->prefix().next;

deallocate_task(*t);

++k;

}

if( my_return_list==plugged_return_list() )

break;

my_free_list = (task*)__TBB_FetchAndStoreW( &my_return_list, (intptr_t)plugged_return_list() );

}

#if __TBB_COUNT_TASK_NODES

my_market->update_task_node_count( my_task_node_count );

#endif /* __TBB_COUNT_TASK_NODES */

// Update my_small_task_count last. Doing so sooner might cause another thread to free *this.

__TBB_ASSERT( my_small_task_count>=k, "my_small_task_count corrupted" );

governor::sign_off(this);

if( __TBB_FetchAndAddW( &my_small_task_count, -k )==k )

NFS_Free( this );

__TBB_ASSERT( t.state()==task::freed, NULL );

generic_scheduler& s = *static_cast<generic_scheduler*>(t.prefix().origin);

__TBB_ASSERT( &s!=this, NULL );

for(;;) {

task* old = s.my_return_list;

if( old==plugged_return_list() )

break;

// Atomically insert t at head of s.my_return_list

t.prefix().next = old;

ITT_NOTIFY( sync_releasing, &s.my_return_list );

if( as_atomic(s.my_return_list).compare_and_swap(&t, old )==old ) {

#if __TBB_PREFETCHING

__TBB_cl_evict(&t.prefix());

__TBB_cl_evict(&t);

#endif

return;

}

}

deallocate_task(t);

if( __TBB_FetchAndDecrementWrelease( &s.my_small_task_count )==1 ) {

// We freed the last task allocated by scheduler s, so it's our responsibility

// to free the scheduler.

NFS_Free( &s );

}

size_t T = __TBB_load_relaxed(my_arena_slot->tail); // mirror

if ( T + num_tasks <= my_arena_slot->my_task_pool_size )

return T;

acquire_task_pool();

size_t H = __TBB_load_relaxed(my_arena_slot->head); // mirror

T -= H;

size_t new_size = T + num_tasks;

__TBB_ASSERT(!my_arena_slot->my_task_pool_size || my_arena_slot->my_task_pool_size >= min_task_pool_size, NULL);

if( !my_arena_slot->my_task_pool_size ) {

__TBB_ASSERT( !in_arena() && !my_arena_slot->task_pool_ptr, NULL );

if( new_size < min_task_pool_size ) new_size = min_task_pool_size;

my_arena_slot->allocate_task_pool( new_size );

}

// If the free space at the beginning of the task pool is too short, we

// are likely facing a pathological single-producer-multiple-consumers

// scenario, and thus it's better to expand the task pool

else if ( new_size <= my_arena_slot->my_task_pool_size - min_task_pool_size/4 ) {

// Relocate the busy part to the beginning of the deque

memmove( my_arena_slot->task_pool_ptr, my_arena_slot->task_pool_ptr + H, T * sizeof(task*) );

my_arena_slot->fill_with_canary_pattern( T, my_arena_slot->tail );

commit_relocated_tasks(T);

}

else {

// Grow task pool. As this operation is rare, and its cost is asymptotically

// amortizable, we can tolerate new task pool allocation done under the lock.

if ( new_size < 2 * my_arena_slot->my_task_pool_size )

new_size = 2 * my_arena_slot->my_task_pool_size;

task** old_pool = my_arena_slot->task_pool_ptr;

my_arena_slot->allocate_task_pool( new_size ); // updates my_task_pool_size

__TBB_ASSERT( T <= my_arena_slot->my_task_pool_size, "new task pool is too short" );

memcpy( my_arena_slot->task_pool_ptr, old_pool + H, T * sizeof(task*) );

commit_relocated_tasks(T);

__TBB_ASSERT( old_pool, "attempt to free NULL TaskPool" );

NFS_Free( old_pool );

}

assert_task_pool_valid();

return T;

if ( !in_arena() )

return; // we are not in arena - nothing to lock

bool sync_prepare_done = false;

for( atomic_backoff b;;b.pause() ) {

#if TBB_USE_ASSERT

__TBB_ASSERT( my_arena_slot == my_arena->my_slots + my_arena_index, "invalid arena slot index" );

// Local copy of the arena slot task pool pointer is necessary for the next

// assertion to work correctly to exclude asynchronous state transition effect.

task** tp = my_arena_slot->task_pool;

__TBB_ASSERT( tp == LockedTaskPool || tp == my_arena_slot->task_pool_ptr, "slot ownership corrupt?" );

#endif

if( my_arena_slot->task_pool != LockedTaskPool &&

as_atomic(my_arena_slot->task_pool).compare_and_swap(LockedTaskPool, my_arena_slot->task_pool_ptr ) == my_arena_slot->task_pool_ptr )

{

// We acquired our own slot

ITT_NOTIFY(sync_acquired, my_arena_slot);

break;

}

else if( !sync_prepare_done ) {

// Start waiting

ITT_NOTIFY(sync_prepare, my_arena_slot);

sync_prepare_done = true;

}

// Someone else acquired a lock, so pause and do exponential backoff.

}

__TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "not really acquired task pool" );

if ( !in_arena() )

return; // we are not in arena - nothing to unlock

__TBB_ASSERT( my_arena_slot, "we are not in arena" );

__TBB_ASSERT( my_arena_slot->task_pool == LockedTaskPool, "arena slot is not locked" );

ITT_NOTIFY(sync_releasing, my_arena_slot);

__TBB_store_with_release( my_arena_slot->task_pool, my_arena_slot->task_pool_ptr );

task** victim_task_pool ) const {

__TBB_ASSERT( victim_arena_slot, "empty victim arena slot pointer" );

__TBB_ASSERT( victim_arena_slot->task_pool == LockedTaskPool, "victim arena slot is not locked" );

ITT_NOTIFY(sync_releasing, victim_arena_slot);

__TBB_store_with_release( victim_arena_slot->task_pool, victim_task_pool );

__TBB_ASSERT( governor::is_set(this), NULL );

if ( &first.prefix().next == &next ) {

// Single task is being spawned

size_t T = prepare_task_pool( 1 );

my_arena_slot->task_pool_ptr[T] = prepare_for_spawning( &first );

commit_spawned_tasks( T + 1 );

}

else {

// Task list is being spawned

task *arr[min_task_pool_size];

fast_reverse_vector<task*> tasks(arr, min_task_pool_size);

task *t_next = NULL;

for( task* t = &first; ; t = t_next ) {

// If t is affinitized to another thread, it may already be executed

// and destroyed by the time prepare_for_spawning returns.

// So milk it while it is alive.

bool end = &t->prefix().next == &next;

t_next = t->prefix().next;

tasks.push_back( prepare_for_spawning(t) );

if( end )

break;

}

size_t num_tasks = tasks.size();

size_t T = prepare_task_pool( num_tasks );

tasks.copy_memory( my_arena_slot->task_pool_ptr + T );

commit_spawned_tasks( T + num_tasks );

}

if ( !in_arena() )

enter_arena();

my_arena->advertise_new_work</*Spawned=*/true>();

assert_task_pool_valid();

__TBB_ASSERT( governor::is_set(this), NULL );

__TBB_ASSERT( &first, NULL );

auto_empty_task dummy( __TBB_CONTEXT_ARG(this, first.prefix().context) );

internal::reference_count n = 0;

for( task* t=&first; ; t=t->prefix().next ) {

++n;

__TBB_ASSERT( !t->prefix().parent, "not a root task, or already running" );

t->prefix().parent = &dummy;

if( &t->prefix().next==&next ) break;

#if __TBB_TASK_GROUP_CONTEXT

__TBB_ASSERT( t->prefix().context == t->prefix().next->prefix().context,

"all the root tasks in list must share the same context");

#endif /* __TBB_TASK_GROUP_CONTEXT */

}

dummy.prefix().ref_count = n+1;

if( n>1 )

local_spawn( *first.prefix().next, next );

local_wait_for_all( dummy, &first );

volatile bool& my_indicator;

auto_indicator ( volatile bool& indicator ) : my_indicator(indicator) { my_indicator = true ;}

~auto_indicator () { my_indicator = false; }

__TBB_ASSERT ( my_arena, "no arena: initialization not completed?" );

__TBB_ASSERT ( my_arena_index < my_arena->my_num_slots, "arena slot index is out-of-bound" );

__TBB_ASSERT ( my_arena_slot == &my_arena->my_slots[my_arena_index], NULL);

__TBB_ASSERT ( my_arena_slot->task_pool == EmptyTaskPool, "someone else grabbed my arena slot?" );

__TBB_ASSERT ( __TBB_load_relaxed(my_arena_slot->head) < __TBB_load_relaxed(my_arena_slot->tail),