MALLOC_ASSERT(isMallocInitialized(),

"Recursion status can be checked only when initialization was done.");

return !mallocRecursionDetected;

MallocMutex bootStrapLock;

Block *bootStrapBlock;

Block *bootStrapBlockUsed;

FreeObject *bootStrapObjectList;

void *allocate(MemoryPool *memPool, size_t size);

void free(void* ptr);

void reset();

static tls_key_t Tid_key;

static intptr_t ThreadIdCount;

unsigned int id;

static ThreadId get() {

ThreadId result;

result.id = reinterpret_cast<intptr_t>(TlsGetValue_func(Tid_key));

if( !result.id ) {

RecursiveMallocCallProtector scoped;

// Thread-local value is zero -> first call from this thread,

// need to initialize with next ID value (IDs start from 1)

result.id = AtomicIncrement(ThreadIdCount); // returned new value!

TlsSetValue_func( Tid_key, reinterpret_cast<void*>(result.id) );

}

return result;

}

static void init() {

#if USE_WINTHREAD

Tid_key = TlsAlloc();

#else

int status = pthread_key_create( &Tid_key, NULL );

if ( status ) {

fprintf (stderr, "The memory manager cannot create tls key during initialization; exiting \n");

exit(1);

}

#endif /* USE_WINTHREAD */

}

static void destroy() {

if( Tid_key ) {

#if USE_WINTHREAD

TlsFree( Tid_key );

#else

int status = pthread_key_delete( Tid_key );

if ( status ) {

fprintf (stderr, "The memory manager cannot delete tls key; exiting \n");

exit(1);

}

#endif /* USE_WINTHREAD */

Tid_key = 0;

}

}

bool isCurrentThreadId() const { return id == ThreadId::get().id; }

void saveCurrentThreadId() { id = ThreadId::get().id; }

#if COLLECT_STATISTICS

static unsigned getMaxThreadId() { return ThreadIdCount; }

friend unsigned int getThreadId();

friend int STAT_increment(ThreadId tid, int bin, int ctr);

#endif

#if USE_PTHREAD

pthread_t tid;

#else

DWORD tid;

#endif

ThreadId() : tid(GetMyTID()) {}

void saveCurrentThreadId() { tid = GetMyTID(); }

#if USE_PTHREAD

bool isCurrentThreadId() const { return pthread_equal(pthread_self(), tid); }

#else

bool isCurrentThreadId() const { return GetCurrentThreadId() == tid; }

#endif

static void init() {}

static void destroy() {}

{

#if USE_WINTHREAD

TLS_pointer_key = TlsAlloc();

#else

int status = pthread_key_create( &TLS_pointer_key, mallocThreadShutdownNotification );

if ( status ) {

fprintf (stderr, "The memory manager cannot create tls key during initialization; exiting \n");

exit(1);

}

#endif /* USE_WINTHREAD */

inline LifoList();

inline void push(Block *block);

inline Block *pop();

Block *top;

#ifdef FINE_GRAIN_LOCKS

MallocMutex lock;

#endif /* FINE_GRAIN_LOCKS */

LifoList bins[numBlockBinLimit];

Block *get(TLSData *tls, unsigned int size);

void put(Bin *bin, Block *block);

void reset();

// if no explicit grainsize, expect to see malloc in user's pAlloc

// and set reasonable low granularity

static const size_t defaultGranularity = estimatedCacheLineSize;

MemoryPool(); // deny

static MallocMutex memPoolListLock;

// list of all active pools is used to release

// all TLS data on thread termination or library unload

MemoryPool *next,

*prev;

ExtMemoryPool extMemPool;

OrphanedBlocks orphanedBlocks;

BootStrapBlocks bootStrapBlocks;

bool init(intptr_t poolId, const MemPoolPolicy* memPoolPolicy);

static void initDefaultPool();

void reset();

void destroy();

void processThreadShutdown(TLSData *tlsData);

inline TLSData *getTLS(bool create);

void clearTLS() { extMemPool.tlsPointerKey.setThreadMallocTLS(NULL); }

Block *getEmptyBlock(size_t size);

void returnEmptyBlock(Block *block, bool poolTheBlock);

// get/put large object to/from local large object cache

void *getFromLLOCache(TLSData *tls, size_t size, size_t alignment);

void putToLLOCache(TLSData *tls, void *object);

FreeObject *publicFreeList;

Block *nextPrivatizable;

Padding<(blockHeaderAlignment -

sizeof(GlobalBlockFields))/sizeof(size_t)> pad_local;

Block *next;

Block *previous; /* Use double linked list to speed up removal */

FreeObject *bumpPtr; /* Bump pointer moves from the end to the beginning of a block */

FreeObject *freeList;

/* Pointer to local data for the owner thread. Used for fast finding tls

TLSData *tlsPtr;

ThreadId ownerTid;

BackRefIdx backRefIdx;

uint16_t allocatedCount; /* Number of objects allocated (obviously by the owning thread) */

uint16_t objectSize;

bool isFull;

friend void *BootStrapBlocks::allocate(MemoryPool *memPool, size_t size);

friend class FreeBlockPool;

friend class StartupBlock;

friend class LifoList;

friend Block *MemoryPool::getEmptyBlock(size_t size);

Padding<(2*blockHeaderAlignment -

sizeof(LocalBlockFields))/sizeof(size_t)> pad_public;

bool empty() const { return allocatedCount==0 && publicFreeList==NULL; }

inline FreeObject* allocate();

inline FreeObject *allocateFromFreeList();

inline bool emptyEnoughToUse();

bool freeListNonNull() { return freeList; }

void freePublicObject(FreeObject *objectToFree);

inline void freeOwnObject(MemoryPool *memPool, TLSData *tls, void *object);

void makeEmpty();

void privatizePublicFreeList();

void restoreBumpPtr();

void privatizeOrphaned(TLSData *tls, unsigned index);

void shareOrphaned(const Bin *bin, unsigned index);

unsigned int getSize() const {

MALLOC_ASSERT(isStartupAllocObject() || objectSize<minLargeObjectSize,

"Invalid object size");

return isStartupAllocObject()? 0 : objectSize;

}

const BackRefIdx *getBackRefIdx() const { return &backRefIdx; }

inline TLSData *ownBlock() const;

bool isStartupAllocObject() const { return objectSize == startupAllocObjSizeMark; }

inline FreeObject *findObjectToFree(void *object) const;

bool checkFreePrecond(void *object) const {

if (allocatedCount>0) {

if (startupAllocObjSizeMark == objectSize) // startup block

return object<=bumpPtr;

else

return allocatedCount <= (slabSize-sizeof(Block))/objectSize

&& (!bumpPtr || object>bumpPtr);

}

return false;

}

const BackRefIdx *getBackRef() const { return &backRefIdx; }

void initEmptyBlock(TLSData *tls, size_t size);

size_t findObjectSize(void *object) const;

void cleanBlockHeader();

static const float emptyEnoughRatio; /* "Reactivate" a block if this share of its objects is free. */

inline FreeObject *allocateFromBumpPtr();

inline FreeObject *findAllocatedObject(const void *address) const;

inline bool isProperlyPlaced(const void *object) const;

inline bool isOrphaned() const { return !tlsPtr; }

inline void markOwned(TLSData *tls) {

ownerTid.saveCurrentThreadId();

tlsPtr = tls;

}

inline void markOrphaned() {

MALLOC_ASSERT(tlsPtr, ASSERT_TEXT);

tlsPtr = NULL;

}

friend class Bin;

friend class TLSData;

friend void MemoryPool::destroy();

Block *activeBlk;

Block *mailbox;

MallocMutex mailLock;

inline Block* getActiveBlock() const { return activeBlk; }

void resetActiveBlock() { activeBlk = 0; }

bool activeBlockUnused() const { return activeBlk && !activeBlk->allocatedCount; }

inline void setActiveBlock(Block *block);

inline Block* setPreviousBlockActive();

Block* getPublicFreeListBlock();

void moveBlockToBinFront(Block *block);

void processLessUsedBlock(MemoryPool *memPool, Block *block);

void outofTLSBin (Block* block);

void verifyTLSBin (size_t size) const;

void pushTLSBin(Block* block);

void verifyInitState() const {

MALLOC_ASSERT( activeBlk == 0, ASSERT_TEXT );

MALLOC_ASSERT( mailbox == 0, ASSERT_TEXT );

}

friend void Block::freePublicObject (FreeObject *objectToFree);

Block *head;

int size;

Backend *backend;

bool lastAccessMiss;

static const int POOL_HIGH_MARK = 32;

static const int POOL_LOW_MARK = 8;

class ResOfGet {

ResOfGet();

public:

Block* block;

bool lastAccMiss;

ResOfGet(Block *b, bool lastMiss) : block(b), lastAccMiss(lastMiss) {}

};

// allocated in zero-initialized memory

FreeBlockPool(Backend *bknd) : backend(bknd) {}

ResOfGet getBlock();

void returnBlock(Block *block);

bool externalCleanup(); // can be called by another thread

static const size_t MAX_TOTAL_SIZE = 4*1024*1024;

// TODO: can single-linked list be faster here?

LargeMemoryBlock *head,

*tail; // need it when do releasing on overflow

size_t totalSize;

int numOfBlocks;

bool put(LargeMemoryBlock *object, ExtMemoryPool *extMemPool);

LargeMemoryBlock *get(size_t size);

bool externalCleanup(ExtMemoryPool *extMemPool);

#if __TBB_MALLOC_WHITEBOX_TEST

LocalLOCImpl() : head(NULL), tail(NULL), totalSize(0), numOfBlocks(0) {}

static size_t getMaxSize() { return MAX_TOTAL_SIZE; }

static const int LOC_HIGH_MARK = HIGH_MARK;

#else

// no ctor, object must be created in zero-initialized memory

#endif

#if USE_PTHREAD

MemoryPool *memPool;

#endif

Bin bin[numBlockBinLimit];

FreeBlockPool freeSlabBlocks;

LocalLOC lloc;

unsigned currCacheIdx;

bool unused;

#if USE_PTHREAD

TLSData(MemoryPool *mPool, Backend *bknd) : memPool(mPool), freeSlabBlocks(bknd) {}

MemoryPool *getMemPool() const { return memPool; }

#else

TLSData(MemoryPool * /*memPool*/, Backend *bknd) : freeSlabBlocks(bknd) {}

#endif

Bin* getAllocationBin(size_t size);

void release(MemoryPool *mPool);

bool externalCleanup(ExtMemoryPool *mPool, bool cleanOnlyUnused) {

if (!unused && cleanOnlyUnused) return false;

// both cleanups to be called, and the order is not important

return lloc.externalCleanup(mPool) | freeSlabBlocks.externalCleanup();

}

bool cleanUnusedActiveBlocks(Backend *backend, bool userPool);

void markUsed() { unused = false; } // called by owner when TLS touched

void markUnused() { unused = true; } // can be called by not owner thread

{

bool released = false;

// active blocks can be not used, so return them to backend

for (uint32_t i=0; i<numBlockBinLimit; i++)

if (bin[i].activeBlockUnused()) {

Block *block = bin[i].getActiveBlock();

bin[i].outofTLSBin(block);

// slab blocks in user's pools do not have valid backRefIdx

if (!userPool)

removeBackRef(*(block->getBackRefIdx()));

backend->putSlabBlock(block);

released = true;

}

return released;

{

bool released = allLocalCaches.cleanup(this, /*cleanOnlyUnused=*/false);

if (TLSData *tlsData = tlsPointerKey.getThreadMallocTLS())

// released only for current thread for now

released |= tlsData->cleanUnusedActiveBlocks(&backend, userPool());

return released;

{

tls->prev = NULL;

MallocMutex::scoped_lock lock(listLock);

MALLOC_ASSERT(head!=tls, ASSERT_TEXT);

tls->next = head;

if (head)

head->prev = tls;

head = tls;

MALLOC_ASSERT(head->next!=head, ASSERT_TEXT);

{

MallocMutex::scoped_lock lock(listLock);

if (head == tls)

head = tls->next;

if (tls->next)

tls->next->prev = tls->prev;

if (tls->prev)

tls->prev->next = tls->next;

MALLOC_ASSERT(!tls->next || tls->next->next!=tls->next, ASSERT_TEXT);

{

bool total = false;

{

MallocMutex::scoped_lock lock(listLock);

for (TLSRemote *curr=head; curr; curr=curr->next)

total |= static_cast<TLSData*>(curr)->

externalCleanup(extPool, cleanOnlyUnused);

}

return total;

{

bool locked;

MallocMutex::scoped_lock lock(listLock, /*block=*/false, &locked);

if (!locked) // not wait for marking if someone doing something with it

return;

for (TLSRemote *curr=head; curr; curr=curr->next)

static_cast<TLSData*>(curr)->markUnused();

{

MALLOC_ASSERT( n>=64 && n<1024, ASSERT_TEXT ); // only needed for bsr array lookup, but always true

unsigned int pos;

#if __ARCH_x86_32||__ARCH_x86_64

# if __linux__||__APPLE__||__FreeBSD__||__NetBSD__||__sun||__MINGW32__

__asm__ ("bsr %1,%0" : "=r"(pos) : "r"(n));

# elif (_WIN32 && (!_WIN64 || __INTEL_COMPILER))

__asm

{

bsr eax, n

mov pos, eax

}

# elif _WIN64 && _MSC_VER>=1400

_BitScanReverse((unsigned long*)&pos, (unsigned long)n);

# else

# error highestBitPos() not implemented for this platform

# endif

#elif __arm__

__asm__ __volatile__

(

"clz %0, %1\n"

"rsb %0, %0, %2\n"

:"=r" (pos) :"r" (n), "I" (31)

);

#else

static unsigned int bsr[16] = {0/*N/A*/,6,7,7,8,8,8,8,9,9,9,9,9,9,9,9};

pos = bsr[ n>>6 ];

#endif /* __ARCH_* */

return pos;

{

if (size <= maxSmallObjectSize) { // selection from 4/8/16/24/32/40/48/56/64

/* Index 0 holds up to 8 bytes, Index 1 16 and so forth */

return indexRequest ? (size - 1) >> 3 : alignUp(size,8);

}

else if (size <= maxSegregatedObjectSize ) { // 80/96/112/128 / 160/192/224/256 / 320/384/448/512 / 640/768/896/1024

unsigned int order = highestBitPos(size-1); // which group of bin sizes?

MALLOC_ASSERT( 6<=order && order<=9, ASSERT_TEXT );

if (indexRequest)

return minSegregatedObjectIndex - (4*6) - 4 + (4*order) + ((size-1)>>(order-2));

else {

unsigned int alignment = 128 >> (9-order); // alignment in the group

MALLOC_ASSERT( alignment==16 || alignment==32 || alignment==64 || alignment==128, ASSERT_TEXT );

return alignUp(size,alignment);

}

}

else {

if( size <= fittingSize3 ) {

if( size <= fittingSize2 ) {

if( size <= fittingSize1 )

return indexRequest ? minFittingIndex : fittingSize1;

else

return indexRequest ? minFittingIndex+1 : fittingSize2;

} else

return indexRequest ? minFittingIndex+2 : fittingSize3;

} else {

if( size <= fittingSize5 ) {

if( size <= fittingSize4 )

return indexRequest ? minFittingIndex+3 : fittingSize4;

else

return indexRequest ? minFittingIndex+4 : fittingSize5;

} else {

MALLOC_ASSERT( 0,ASSERT_TEXT ); // this should not happen

return ~0U;

}

}

}

{

MALLOC_ASSERT( ptr, ASSERT_TEXT );

{ // Lock with acquire

MallocMutex::scoped_lock scoped_cs(bootStrapLock);

((FreeObject*)ptr)->next = bootStrapObjectList;

bootStrapObjectList = (FreeObject*)ptr;

} // Unlock with release

{

bootStrapBlock = bootStrapBlockUsed = NULL;

bootStrapObjectList = NULL;

{

// MallocMutex assumes zero initialization

memset(&lock, 0, sizeof(MallocMutex));

{

MallocMutex::scoped_lock scoped_cs(lock);

block->next = top;

top = block;

LargeMemoryBlock *next, *lmb = loHead;

loHead = NULL;

for (; lmb; lmb = next) {

next = lmb->gNext;

if (poolDestroy) {

// as it's pool destruction, no need to return object to backend,

// only remove backrefs, as they are global

removeBackRef(lmb->backRefIdx);

} else {

// clean g(Next|Prev) to prevent removing lmb

// from AllLargeBlocksList inside returnLargeObject

lmb->gNext = lmb->gPrev = NULL;

backend->returnLargeObject(lmb);

}

}