{

const void * m_ptr;

char * m_name;

Uint32 m_contended_count;

Uint32 m_spin_count;

{

return syscall(SYS_futex,

addr, FUTEX_WAIT, val, timeout, 0, 0) == 0 ? 0 : errno;

{

volatile unsigned m_futex_state;

enum {

FS_RUNNING = 0,

FS_SLEEPING = 1

};

thr_wait() { xcng(&m_futex_state, FS_RUNNING);}

void init () {}

{

volatile unsigned * val = &wait->m_futex_state;

/**

if (xcng(val, thr_wait::FS_RUNNING) == thr_wait::FS_SLEEPING)

{

return futex_wake(val);

}

return 0;

{

bool m_need_wakeup;

NdbMutex *m_mutex;

NdbCondition *m_cond;

thr_wait() : m_need_wakeup(false), m_mutex(0), m_cond(0) {}

void init() {

m_mutex = NdbMutex_Create();

m_cond = NdbCondition_Create();

}

bool (*check_callback)(struct thr_data *), struct thr_data *check_arg)

{

struct timespec end;

NdbCondition_ComputeAbsTime(&end, nsec/1000000);

NdbMutex_Lock(wait->m_mutex);

Uint32 waits = 0;

/* May have spurious wakeups: Always recheck condition predicate */

while ((*check_callback)(check_arg))

{

wait->m_need_wakeup = true;

waits++;

if (NdbCondition_WaitTimeoutAbs(wait->m_cond,

wait->m_mutex, &end) == ETIMEDOUT)

{

wait->m_need_wakeup = false;

break;

}

}

NdbMutex_Unlock(wait->m_mutex);

return (waits > 0);

{

NdbMutex_Lock(wait->m_mutex);

// We should avoid signaling when not waiting for wakeup

if (wait->m_need_wakeup)

{

wait->m_need_wakeup = false;

NdbCondition_Signal(wait->m_cond);

}

NdbMutex_Unlock(wait->m_mutex);

return 0;

{

thr_spin_lock(const char * name = 0)

{

m_lock = 0;

register_lock(this, name);

}

union {

volatile Uint32 m_lock;

char pad[SZ];

};

{

mt_lock_stat* s = lookup_lock(sl); // lookup before owning lock

loop:

Uint32 spins = 0;

do {

spins++;

cpu_pause();

} while (* val == 1);

if (unlikely(xcng(val, 1) != 0))

goto loop;

if (s)

{

s->m_spin_count += spins;

Uint32 count = ++s->m_contended_count;

Uint32 freq = (count > 10000 ? 5000 : (count > 20 ? 200 : 1));

if ((count % freq) == 0)

printf("%s waiting for lock, contentions: %u spins: %u\n",

s->m_name, count, s->m_spin_count);

}

{

volatile unsigned* val = &sl->m_lock;

if (likely(xcng(val, 1) == 0))

return;

lock_slow(sl, val);

{

/**

mb();

sl->m_lock = 0;

{

volatile unsigned* val = &sl->m_lock;

return xcng(val, 1);

{

thr_mutex(const char * name = 0) {

NdbMutex_Init(&m_mutex);

register_lock(this, name);

}

union {

NdbMutex m_mutex;

char pad[SZ];

};

{

thr_safe_pool(const char * name) : m_free_list(0), m_cnt(0), m_lock(name) {}

T* m_free_list;

Uint32 m_cnt;

thr_spin_lock<NDB_CL - (sizeof(void*) + sizeof(Uint32))> m_lock;

T* seize(Ndbd_mem_manager *mm, Uint32 rg) {

T* ret = 0;

lock(&m_lock);

if (m_free_list)

{

assert(m_cnt);

m_cnt--;

ret = m_free_list;

m_free_list = ret->m_next;

unlock(&m_lock);

}

else

{

Uint32 dummy;

unlock(&m_lock);

ret = reinterpret_cast<T*>

(mm->alloc_page(rg, &dummy,

Ndbd_mem_manager::NDB_ZONE_ANY));

// ToDo: How to deal with failed allocation?!?

// I think in this case we need to start grabbing buffers kept for signal

// trace.

}

return ret;

}

void release(Ndbd_mem_manager *mm, Uint32 rg, T* t) {

lock(&m_lock);

t->m_next = m_free_list;

m_free_list = t;

m_cnt++;

unlock(&m_lock);

}

void release_list(Ndbd_mem_manager *mm, Uint32 rg,

T* head, T* tail, Uint32 cnt) {

lock(&m_lock);

tail->m_next = m_free_list;

m_free_list = head;

m_cnt += cnt;

unlock(&m_lock);

}

{

thread_local_pool(thr_safe_pool<T> *global_pool, unsigned max_free) :

m_max_free(max_free),

m_free(0),

m_freelist(0),

m_global_pool(global_pool)

{

}

T *seize(Ndbd_mem_manager *mm, Uint32 rg) {

T *tmp = m_freelist;

if (tmp)

{

m_freelist = tmp->m_next;

assert(m_free > 0);

m_free--;

}

else

tmp = m_global_pool->seize(mm, rg);

validate();

return tmp;

}

void release(Ndbd_mem_manager *mm, Uint32 rg, T *t) {

unsigned free = m_free;

if (free < m_max_free)

{

m_free = free + 1;

t->m_next = m_freelist;

m_freelist = t;

}

else

m_global_pool->release(mm, rg, t);

validate();

}

/**

void release_local(T *t) {

m_free++;

t->m_next = m_freelist;

m_freelist = t;

validate();

}

void validate() const {

#ifdef VM_TRACE

Uint32 cnt = 0;

T* t = m_freelist;

while (t)

{

cnt++;

t = t->m_next;

}

assert(cnt == m_free);

#endif

}

/**

void release_global(Ndbd_mem_manager *mm, Uint32 rg) {

validate();

unsigned cnt = 0;

unsigned free = m_free;

Uint32 maxfree = m_max_free;

assert(maxfree > 0);

T* head = m_freelist;

T* tail = m_freelist;

if (free > maxfree)

{

cnt++;

free--;

while (free > maxfree)

{

cnt++;

free--;

tail = tail->m_next;

}

assert(free == maxfree);

m_free = free;

m_freelist = tail->m_next;

m_global_pool->release_list(mm, rg, head, tail, cnt);

}

validate();

}

void release_all(Ndbd_mem_manager *mm, Uint32 rg) {

validate();

T* head = m_freelist;

T* tail = m_freelist;

if (tail)

{

unsigned cnt = 1;

while (tail->m_next != 0)

{

cnt++;

tail = tail->m_next;

}

m_global_pool->release_list(mm, rg, head, tail, cnt);

m_free = 0;

m_freelist = 0;

}

validate();

}

void set_pool(thr_safe_pool<T> * pool) { m_global_pool = pool; }

unsigned m_max_free;

unsigned m_free;

T *m_freelist;

thr_safe_pool<T> *m_global_pool;

{

static const unsigned SIZE = 8190;

/*

Uint32 m_len;

/*

Uint32 m_prioa;

union {

Uint32 m_data[SIZE];

thr_job_buffer * m_next; // For free-list

};

{

unsigned m_read_index; // Read/written by consumer, read by producer

unsigned m_write_index; // Read/written by producer, read by consumer

Uint32 used() const;

{

static const unsigned SIZE = 31;

struct thr_job_queue_head* m_head;

struct thr_job_buffer* m_buffers[SIZE];

{

/*

Uint32 m_write_index;

Uint32 m_write_pos;

/* Thread-local copy of thr_job_queue::m_buffers[m_write_index]. */

thr_job_buffer *m_write_buffer;

/* Number of signals inserted since last flush to thr_job_queue. */

Uint32 m_pending_signals;

/* Number of signals inserted since last wakeup */

Uint32 m_pending_signals_wakeup;

{

/*

Uint32 m_read_index;

/*

Uint32 m_read_pos;

/*

thr_job_buffer *m_read_buffer;

/*

Uint32 m_read_end; // End within current thr_job_buffer. (*m_read_buffer)

Uint32 m_write_index; // Last available thr_job_buffer.

bool is_empty() const

{

assert(m_read_index != m_write_index || m_read_pos <= m_read_end);

return (m_read_index == m_write_index) && (m_read_pos >= m_read_end);

}

{

static const unsigned SQ_SIZE = 512;

static const unsigned LQ_SIZE = 512;

static const unsigned PAGES = 32 * (SQ_SIZE + LQ_SIZE) / 8192;

Uint32 * m_delayed_signals[PAGES];

Uint32 m_next_free;

Uint32 m_next_timer;

Uint32 m_current_time;

Uint32 m_cnt[2];

Uint32 m_short_queue[SQ_SIZE];

Uint32 m_long_queue[LQ_SIZE];

{

static const Uint32 PGSIZE = 32768;

#if SIZEOF_CHARP == 4

static const Uint32 HEADER_SIZE = 8;

#else

static const Uint32 HEADER_SIZE = 12;

#endif

static Uint32 max_bytes() {

return PGSIZE - offsetof(thr_send_page, m_data);

}

/* Next page */

thr_send_page* m_next;

/* Bytes of send data available in this page. */

Uint16 m_bytes;

/* Start of unsent data */

Uint16 m_start;

/* Data; real size is to the end of one page. */

char m_data[2];

{

thr_send_page* m_first_page;

thr_send_page* m_last_page;

{

unsigned m_write_index;

#if SIZEOF_CHARP == 8

unsigned m_unused;

thr_send_page* m_buffers[7];

static const unsigned SIZE = 7;

#else

thr_send_page* m_buffers[15];

static const unsigned SIZE = 15;

#endif

{

thr_data() : m_jba_write_lock("jbalock"),

m_send_buffer_pool(0, THR_FREE_BUF_MAX) {}

thr_wait m_waiter;

unsigned m_thr_no;

/**

unsigned m_max_signals_per_jb;

/**

unsigned m_max_exec_signals;

Uint64 m_time;

struct thr_tq m_tq;

/* Prio A signal incoming queue. */

struct thr_spin_lock<64> m_jba_write_lock;

struct thr_job_queue m_jba;

struct thr_job_queue_head m_jba_head;

/* Thread-local read state of prio A buffer. */

struct thr_jb_read_state m_jba_read_state;

/*

/*

struct thr_job_buffer* m_next_buffer;

/*

struct thr_job_buffer *m_free_fifo[THR_FREE_BUF_MAX];

/* m_first_free is the index of the entry to return next from seize(). */

Uint32 m_first_free;

/* m_first_unused is the first unused entry in m_free_fifo. */

Uint32 m_first_unused;

/*

struct thr_job_queue_head m_in_queue_head[MAX_THREADS];

struct thr_job_queue m_in_queue[MAX_THREADS];

/* These are the write states of m_in_queue[self] in each thread. */

struct thr_jb_write_state m_write_states[MAX_THREADS];

/* These are the read states of all of our own m_in_queue[]. */

struct thr_jb_read_state m_read_states[MAX_THREADS];

/* Jam buffers for making trace files at crashes. */

EmulatedJamBuffer m_jam;

/* Watchdog counter for this thread. */

Uint32 m_watchdog_counter;

/* Signal delivery statistics. */

Uint32 m_prioa_count;

Uint32 m_prioa_size;

Uint32 m_priob_count;

Uint32 m_priob_size;

/* Array of node ids with pending remote send data. */

Uint8 m_pending_send_nodes[MAX_NTRANSPORTERS];

/* Number of node ids in m_pending_send_nodes. */

Uint32 m_pending_send_count;

/**

Bitmask<(MAX_NTRANSPORTERS+31)/32> m_pending_send_mask;

/* pool for send buffers */

class thread_local_pool<thr_send_page> m_send_buffer_pool;

/* Send buffer for this thread, these are not touched by any other thread */

struct thr_send_buffer m_send_buffers[MAX_NTRANSPORTERS];

/* Block instances (main and worker) handled by this thread. */

/* Used for sendpacked (send-at-job-buffer-end). */

Uint32 m_instance_count;

BlockNumber m_instance_list[MAX_INSTANCES_PER_THREAD];

SectionSegmentPool::Cache m_sectionPoolCache;

Uint32 m_cpu;

pthread_t m_thr_id;

NdbThread* m_thread;

{

struct thr_data * m_selfptr;

mt_send_handle(thr_data* ptr) : m_selfptr(ptr) {}

virtual ~mt_send_handle() {}

virtual Uint32 *getWritePtr(NodeId node, Uint32 len, Uint32 prio, Uint32 max);

virtual Uint32 updateWritePtr(NodeId node, Uint32 lenBytes, Uint32 prio);

virtual bool forceSend(NodeId node);

{

trp_callback() {}

/* Callback interface. */

int checkJobBuffer();

void reportSendLen(NodeId nodeId, Uint32 count, Uint64 bytes);

void lock_transporter(NodeId node);

void unlock_transporter(NodeId node);

Uint32 get_bytes_to_send_iovec(NodeId node, struct iovec *dst, Uint32 max);

Uint32 bytes_sent(NodeId node, Uint32 bytes);

bool has_data_to_send(NodeId node);

void reset_send_buffer(NodeId node, bool should_be_empty);

{

thr_repository()

: m_receive_lock("recvlock"),

m_section_lock("sectionlock"),

m_mem_manager_lock("memmanagerlock"),

m_jb_pool("jobbufferpool"),

m_sb_pool("sendbufferpool")

{}

struct thr_spin_lock<64> m_receive_lock;

struct thr_spin_lock<64> m_section_lock;

struct thr_spin_lock<64> m_mem_manager_lock;

struct thr_safe_pool<thr_job_buffer> m_jb_pool;

struct thr_safe_pool<thr_send_page> m_sb_pool;

Ndbd_mem_manager * m_mm;

unsigned m_thread_count;

struct thr_data m_thread[MAX_THREADS];

/**

/* The buffers that are to be sent */

struct send_buffer

{

/**

struct thr_spin_lock<8> m_send_lock;

/**

struct thr_send_buffer m_buffer;

/**

Uint32 m_force_send;

/**

Uint32 m_send_thread;

/**

Uint32 m_bytes;

/* read index(es) in thr_send_queue */

Uint32 m_read_index[MAX_THREADS];