#include <string.h> /* for memset */

#include <errno.h>

#include "tbbmalloc_internal.h"

namespace rml {

namespace internal {

#if USE_DEFAULT_MEMORY_MAPPING

#include "MapMemory.h"

#else

#endif

void* getRawMemory (size_t size, bool hugePages) {

return MapMemory(size, hugePages);

}

int freeRawMemory (void *object, size_t size) {

return UnmapMemory(object, size);

}

void HugePagesStatus::registerAllocation(bool gotPage)

}

void HugePagesStatus::registerReleasing(void* addr, size_t size)

}

void HugePagesStatus::printStatus() {

}

void HugePagesStatus::doPrintStatus(bool state, const char *stateName)

}

#if CHECK_ALLOCATION_RANGE

void Backend::UsedAddressRange::registerAlloc(uintptr_t left, uintptr_t right)

{

MallocMutex::scoped_lock lock(mutex);

if (left < leftBound)

leftBound = left;

if (right > rightBound)

rightBound = right;

MALLOC_ASSERT(leftBound, ASSERT_TEXT);

MALLOC_ASSERT(leftBound < rightBound, ASSERT_TEXT);

MALLOC_ASSERT(leftBound <= left && right <= rightBound, ASSERT_TEXT);

}

void Backend::UsedAddressRange::registerFree(uintptr_t left, uintptr_t right)

{

MallocMutex::scoped_lock lock(mutex);

if (leftBound == left) {

if (rightBound == right) {

leftBound = ADDRESS_UPPER_BOUND;

rightBound = 0;

} else

leftBound = right;

} else if (rightBound == right)

rightBound = left;

MALLOC_ASSERT((!rightBound && leftBound == ADDRESS_UPPER_BOUND)

|| leftBound < rightBound, ASSERT_TEXT);

}

#endif // CHECK_ALLOCATION_RANGE

void *Backend::allocRawMem(size_t &size)

{

void *res = NULL;

size_t allocSize;

if (extMemPool->userPool()) {

if (extMemPool->fixedPool && bootsrapMemDone==FencedLoad(bootsrapMemStatus))

return NULL;

MALLOC_ASSERT(bootsrapMemStatus!=bootsrapMemNotDone,

"Backend::allocRawMem() called prematurely?");

// TODO: support for raw mem not aligned at sizeof(uintptr_t)

// memory from fixed pool is asked once and only once

allocSize = alignUpGeneric(size, extMemPool->granularity);

res = (*extMemPool->rawAlloc)(extMemPool->poolId, allocSize);

} else {

// check if alignment to huge page size is recommended

size_t hugePageSize = hugePages.recommendedGranularity();

allocSize = alignUpGeneric(size, hugePageSize? hugePageSize : extMemPool->granularity);

// try to get them at 1st allocation and still use, if successful

// if 1st try is unsuccessful, no more trying

if (FencedLoad(hugePages.enabled)) {

MALLOC_ASSERT(hugePageSize, "Inconsistent state of HugePagesStatus");

res = getRawMemory(allocSize, /*hugePages=*/true);

hugePages.registerAllocation(res);

}

if (!res)

res = getRawMemory(allocSize, /*hugePages=*/false);

}

if (res) {

MALLOC_ASSERT(allocSize > 0, "Invalid size of an allocated region.");

size = allocSize;

if (!extMemPool->userPool())

usedAddrRange.registerAlloc((uintptr_t)res, (uintptr_t)res+size);

#if MALLOC_DEBUG

volatile size_t curTotalSize = totalMemSize; // to read global value once

MALLOC_ASSERT(curTotalSize+size > curTotalSize, "Overflow allocation size.");

#endif

AtomicAdd((intptr_t&)totalMemSize, size);

}

return res;

}

bool Backend::freeRawMem(void *object, size_t size)

}

class GuardedSize : tbb::internal::no_copy {

};

struct MemRegion {

};

class BlockMutexes {

};

class FreeBlock : BlockMutexes {

};

struct LastFreeBlock : public FreeBlock {

MemRegion *memRegion;

};

const size_t FreeBlock::minBlockSize = sizeof(FreeBlock);

inline bool BackendSync::waitTillBlockReleased(intptr_t startModifiedCnt)

}

void CoalRequestQ::putBlock(FreeBlock *fBlock)

}

FreeBlock *CoalRequestQ::getAll()

{

for (;;) {

FreeBlock *myBlToFree = (FreeBlock*)FencedLoad((intptr_t&)blocksToFree);

if (!myBlToFree)

return NULL;

else {

if (myBlToFree ==

(FreeBlock*)AtomicCompareExchange((intptr_t&)blocksToFree,

0, (intptr_t)myBlToFree))

return myBlToFree;

else

continue;

}

}

}

inline void CoalRequestQ::blockWasProcessed()

}

FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync,

size_t size, bool needAlignedRes, bool alignedBin, bool wait,

int *binLocked)

{

Bin *b = &freeBins[binIdx];

try_next:

FreeBlock *fBlock = NULL;

if (b->head) {

bool locked;

MallocMutex::scoped_lock scopedLock(b->tLock, wait, &locked);

if (!locked) {

if (binLocked) (*binLocked)++;

return NULL;

}

for (FreeBlock *curr = b->head; curr; curr = curr->next) {

size_t szBlock = curr->tryLockBlock();

if (!szBlock) {

// block is locked, re-do bin lock, as there is no place to spin

// while block coalescing

goto try_next;

}

if (alignedBin || !needAlignedRes) {

size_t splitSz = szBlock - size;

// If we got a block as split result,

// it must have a room for control structures.

if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize ||

!splitSz))

fBlock = curr;

} else {

void *newB = alignUp(curr, slabSize);

uintptr_t rightNew = (uintptr_t)newB + size;

uintptr_t rightCurr = (uintptr_t)curr + szBlock;

// appropriate size, and left and right split results

// are either big enough or non-existent

if (rightNew <= rightCurr

&& (newB==curr ||

(uintptr_t)newB-(uintptr_t)curr >= FreeBlock::minBlockSize)

&& (rightNew==rightCurr ||

rightCurr - rightNew >= FreeBlock::minBlockSize))

fBlock = curr;

}

if (fBlock) {

// consume must be called before result of removing from a bin

// is visible externally.

sync->blockConsumed();

if (alignedBin && needAlignedRes &&

Backend::sizeToBin(szBlock-size) == Backend::sizeToBin(szBlock)) {

// free remainder of fBlock stay in same bin,

// so no need to remove it from the bin

// TODO: add more "still here" cases

FreeBlock *newFBlock = fBlock;

// return block from right side of fBlock

fBlock = (FreeBlock*)((uintptr_t)newFBlock + szBlock - size);

MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block");

fBlock->initHeader();

fBlock->setLeftFree(szBlock - size);

newFBlock->setMeFree(szBlock - size);

fBlock->sizeTmp = size;

} else {

b->removeBlock(fBlock);

if (freeBins[binIdx].empty())

bitMask.set(binIdx, false);

fBlock->sizeTmp = szBlock;

}

break;

} else { // block size is not valid, search for next block in the bin

curr->setMeFree(szBlock);

curr->rightNeig(szBlock)->setLeftFree(szBlock);

}

}

}

return fBlock;

}

bool Backend::IndexedBins::tryReleaseRegions(int binIdx, Backend *backend)

{

Bin *b = &freeBins[binIdx];

FreeBlock *fBlockList = NULL;

// got all blocks from the bin and re-do coalesce on them

// to release single-block regions

try_next:

if (b->head) {

MallocMutex::scoped_lock binLock(b->tLock);

for (FreeBlock *curr = b->head; curr; ) {

size_t szBlock = curr->tryLockBlock();

if (!szBlock)

goto try_next;

FreeBlock *next = curr->next;

b->removeBlock(curr);

curr->sizeTmp = szBlock;

curr->nextToFree = fBlockList;

fBlockList = curr;

curr = next;

}

}

return backend->coalescAndPutList(fBlockList, /*forceCoalescQDrop=*/true,

/*reportBlocksProcessed=*/false);

}

void Backend::Bin::removeBlock(FreeBlock *fBlock)

{

MALLOC_ASSERT(fBlock->next||fBlock->prev||fBlock==head,

"Detected that a block is not in the bin.");

if (head == fBlock)

head = fBlock->next;

if (tail == fBlock)

tail = fBlock->prev;

if (fBlock->prev)

fBlock->prev->next = fBlock->next;

if (fBlock->next)

fBlock->next->prev = fBlock->prev;

}

void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t blockSz, bool addToTail)

{

Bin *b = &freeBins[binIdx];

fBlock->myBin = binIdx;

fBlock->aligned = toAlignedBin(fBlock, blockSz);

fBlock->next = fBlock->prev = NULL;

{

MallocMutex::scoped_lock scopedLock(b->tLock);

if (addToTail) {

fBlock->prev = b->tail;

b->tail = fBlock;

if (fBlock->prev)

fBlock->prev->next = fBlock;

if (!b->head)

b->head = fBlock;

} else {

fBlock->next = b->head;

b->head = fBlock;

if (fBlock->next)

fBlock->next->prev = fBlock;

if (!b->tail)

b->tail = fBlock;

}

}

bitMask.set(binIdx, true);

}

bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail)

{

bool locked;

Bin *b = &freeBins[binIdx];

fBlock->myBin = binIdx;

fBlock->aligned = toAlignedBin(fBlock, fBlock->sizeTmp);

if (addToTail) {

fBlock->next = NULL;

{

MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);

if (!locked)

return false;

fBlock->prev = b->tail;

b->tail = fBlock;

if (fBlock->prev)

fBlock->prev->next = fBlock;

if (!b->head)

b->head = fBlock;

}

} else {

fBlock->prev = NULL;

{

MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);

if (!locked)

return false;

fBlock->next = b->head;

b->head = fBlock;

if (fBlock->next)

fBlock->next->prev = fBlock;

if (!b->tail)

b->tail = fBlock;

}

}

bitMask.set(binIdx, true);

return true;

}

void Backend::IndexedBins::reset()

{

for (int i=0; i<Backend::freeBinsNum; i++)

freeBins[i].reset();

bitMask.reset();

}

void Backend::IndexedBins::lockRemoveBlock(int binIdx, FreeBlock *fBlock)

{

MallocMutex::scoped_lock scopedLock(freeBins[binIdx].tLock);

freeBins[binIdx].removeBlock(fBlock);

if (freeBins[binIdx].empty())

bitMask.set(binIdx, false);

}

bool ExtMemoryPool::regionsAreReleaseable() const

{

return !keepAllMemory && !delayRegsReleasing;

}

FreeBlock *Backend::splitUnalignedBlock(FreeBlock *fBlock, int num, size_t size,

bool needAlignedBlock)

{

const size_t totalSize = num*size;

if (needAlignedBlock) {

size_t fBlockSz = fBlock->sizeTmp;

uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlockSz;

FreeBlock *newB = alignUp(fBlock, slabSize);

FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newB + totalSize);

// Space to use is in the middle,

// ... return free right part

if ((uintptr_t)rightPart != fBlockEnd) {

rightPart->initHeader(); // to prevent coalescing rightPart with fBlock

coalescAndPut(rightPart, fBlockEnd - (uintptr_t)rightPart);

}

// ... and free left part

if (newB != fBlock) {

newB->initHeader(); // to prevent coalescing fBlock with newB

coalescAndPut(fBlock, (uintptr_t)newB - (uintptr_t)fBlock);

}

fBlock = newB;

MALLOC_ASSERT(isAligned(fBlock, slabSize), ASSERT_TEXT);

} else {

if (size_t splitSz = fBlock->sizeTmp - totalSize) {

// split block and return free right part

FreeBlock *splitB = (FreeBlock*)((uintptr_t)fBlock + totalSize);

splitB->initHeader();

coalescAndPut(splitB, splitSz);

}

}

FreeBlock::markBlocks(fBlock, num, size);

return fBlock;

}

FreeBlock *Backend::splitAlignedBlock(FreeBlock *fBlock, int num, size_t size,

bool needAlignedBlock)

{

if (fBlock->sizeTmp != num*size) { // i.e., need to split the block

FreeBlock *newAlgnd;

size_t newSz;

if (needAlignedBlock) {

newAlgnd = fBlock;

fBlock = (FreeBlock*)((uintptr_t)newAlgnd + newAlgnd->sizeTmp

- num*size);

MALLOC_ASSERT(isAligned(fBlock, slabSize), "Invalid free block");

fBlock->initHeader();

newSz = newAlgnd->sizeTmp - num*size;

} else {

newAlgnd = (FreeBlock*)((uintptr_t)fBlock + num*size);

newSz = fBlock->sizeTmp - num*size;

newAlgnd->initHeader();

}

coalescAndPut(newAlgnd, newSz);

}

MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize),

"Expect to get aligned block, if one was requested.");

FreeBlock::markBlocks(fBlock, num, size);

return fBlock;

}

inline size_t Backend::getMaxBinnedSize() const

}

inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq,

size_t requestSize) const

{

return requestSize > oldMaxReq && requestSize < backend->getMaxBinnedSize();

}

FreeBlock *Backend::releaseMemInCaches(intptr_t startModifiedCnt,

int *lockedBinsThreshold, int numOfLockedBins)

{

// something released from caches

if (extMemPool->hardCachesCleanup()

// ..or can use blocks that are in processing now

|| bkndSync.waitTillBlockReleased(startModifiedCnt))

return (FreeBlock*)VALID_BLOCK_IN_BIN;

// OS can't give us more memory, but we have some in locked bins

if (*lockedBinsThreshold && numOfLockedBins) {

*lockedBinsThreshold = 0;

return (FreeBlock*)VALID_BLOCK_IN_BIN;

}

return NULL; // nothing found, give up

}

FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt,

int *lockedBinsThreshold, int numOfLockedBins,

bool *splittableRet)

{

FreeBlock *block;

// The block sizes can be divided into 3 groups:

// 1. "quite small": popular object size, we are in bootstarp or something

// like; request several regions.

// 2. "quite large": we want to have several such blocks in the region

// but not want several pre-allocated regions.

// 3. "huge": exact fit, we allocate only one block and do not allow

// any other allocations to placed in a region.

// Dividing the block sizes in these groups we are trying to balance between

// too small regions (that leads to fragmentation) and too large ones (that

// leads to excessive address space consumption). If a region is "too

// large", allocate only one, to prevent fragmentation. It supposedly

// doesn't hurt performance, because the object requested by user is large.

// Bounds for the groups are:

const size_t maxBinned = getMaxBinnedSize();

const size_t quiteSmall = maxBinned / 8;

const size_t quiteLarge = maxBinned;

if (blockSize >= quiteLarge) {

// Do not interact with other threads via semaphores, as for exact fit

// we can't share regions with them, memory requesting is individual.

block = addNewRegion(blockSize, MEMREG_ONE_BLOCK, /*addToBin=*/false);

if (!block)

return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins);

*splittableRet = false;

} else {

const size_t regSz_sizeBased = alignUp(4*maxRequestedSize, 1024*1024);

// Another thread is modifying backend while we can't get the block.

// Wait while it leaves and re-do the scan

// before trying other ways to extend the backend.

if (bkndSync.waitTillBlockReleased(startModifiedCnt)

// semaphore is protecting adding more more memory from OS

|| memExtendingSema.wait())

return (FreeBlock*)VALID_BLOCK_IN_BIN;

if (startModifiedCnt != bkndSync.getNumOfMods()) {

memExtendingSema.signal();

return (FreeBlock*)VALID_BLOCK_IN_BIN;

}

if (blockSize < quiteSmall) {

// For this size of blocks, add NUM_OF_REG "advance" regions in bin,

// and return one as a result.

// TODO: add to bin first, because other threads can use them right away.

// This must be done carefully, because blocks in bins can be released

// in releaseCachesToLimit().

const unsigned NUM_OF_REG = 3;

block = addNewRegion(regSz_sizeBased, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/false);

if (block)

for (unsigned idx=0; idx<NUM_OF_REG; idx++)

if (! addNewRegion(regSz_sizeBased, MEMREG_FLEXIBLE_SIZE, /*addToBin=*/true))

break;

} else {

block = addNewRegion(regSz_sizeBased, MEMREG_SEVERAL_BLOCKS, /*addToBin=*/false);

}

memExtendingSema.signal();

// no regions found, try to clean cache

if (!block || block == (FreeBlock*)VALID_BLOCK_IN_BIN)

return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins);

// Since a region can hold more than one block it can be splitted.

*splittableRet = true;

}

// after asking memory from OS, release caches if we above the memory limits

releaseCachesToLimit();

return block;

}

void Backend::releaseCachesToLimit()

}

FreeBlock *Backend::IndexedBins::

findBlock(int nativeBin, BackendSync *sync, size_t size,

bool resSlabAligned, bool alignedBin, int *numOfLockedBins)

{

for (int i=getMinNonemptyBin(nativeBin); i<freeBinsNum; i=getMinNonemptyBin(i+1))

if (FreeBlock *block = getFromBin(i, sync, size, resSlabAligned, alignedBin,

/*wait=*/false, numOfLockedBins))

return block;

return NULL;

}

void Backend::requestBootstrapMem()

}

FreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock)

{

FreeBlock *block = NULL;

const size_t totalReqSize = num*size;

// no splitting after requesting new region, asks exact size

const int nativeBin = sizeToBin(totalReqSize);

requestBootstrapMem();

// If we found 2 or less locked bins, it's time to ask more memory from OS.

// But nothing can be asked from fixed pool. And we prefer wait, not ask

// for more memory, if block is quite large.

int lockedBinsThreshold = extMemPool->fixedPool || size>=maxBinned_SmallPage? 0 : 2;

// Find maximal requested size limited by getMaxBinnedSize()

AtomicUpdate(maxRequestedSize, totalReqSize, MaxRequestComparator(this));

scanCoalescQ(/*forceCoalescQDrop=*/false);

bool splittable = true;

for (;;) {

const intptr_t startModifiedCnt = bkndSync.getNumOfMods();

int numOfLockedBins;

do {

numOfLockedBins = 0;

// TODO: try different bin search order

if (needAlignedBlock) {

block = freeAlignedBins.findBlock(nativeBin, &bkndSync, num*size,

/*needAlignedBlock=*/true, /*alignedBin=*/true,

&numOfLockedBins);

if (!block)

block = freeLargeBins.findBlock(nativeBin, &bkndSync, num*size,

/*needAlignedBlock=*/true, /*alignedBin=*/false,

&numOfLockedBins);

} else {

block = freeLargeBins.findBlock(nativeBin, &bkndSync, num*size,

/*needAlignedBlock=*/false, /*alignedBin=*/false,

&numOfLockedBins);

if (!block)

block = freeAlignedBins.findBlock(nativeBin, &bkndSync, num*size,

/*needAlignedBlock=*/false, /*alignedBin=*/true,

&numOfLockedBins);

}

} while (!block && numOfLockedBins>lockedBinsThreshold);

if (block)

break;

if (!(scanCoalescQ(/*forceCoalescQDrop=*/true)

| extMemPool->softCachesCleanup())) {

// bins are not updated,

// only remaining possibility is to ask for more memory

block =

askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold,

numOfLockedBins, &splittable);

if (!block)

return NULL;

if (block != (FreeBlock*)VALID_BLOCK_IN_BIN) {

// size can be increased in askMemFromOS, that's why >=

MALLOC_ASSERT(block->sizeTmp >= size, ASSERT_TEXT);

break;

}

// valid block somewhere in bins, let's find it

block = NULL;

}

}

MALLOC_ASSERT(block, ASSERT_TEXT);

if (splittable)

block = toAlignedBin(block, block->sizeTmp)?

splitAlignedBlock(block, num, size, needAlignedBlock) :

splitUnalignedBlock(block, num, size, needAlignedBlock);

// matched blockConsumed() from startUseBlock()

bkndSync.blockReleased();

return block;

}

LargeMemoryBlock *Backend::getLargeBlock(size_t size)

{

LargeMemoryBlock *lmb =

(LargeMemoryBlock*)genericGetBlock(1, size, /*needAlignedRes=*/false);

if (lmb) {

lmb->unalignedSize = size;

if (extMemPool->userPool())

extMemPool->lmbList.add(lmb);

}

return lmb;

}

void *Backend::getBackRefSpace(size_t size, bool *rawMemUsed)

{

// This block is released only at shutdown, so it can prevent

// a entire region releasing when it's received from the backend,

// so prefer getRawMemory using.

if (void *ret = getRawMemory(size, /*hugePages=*/false)) {

*rawMemUsed = true;

return ret;

}

void *ret = genericGetBlock(1, size, /*needAlignedRes=*/false);

if (ret) *rawMemUsed = false;

return ret;

}

void Backend::putBackRefSpace(void *b, size_t size, bool rawMemUsed)

}

void Backend::removeBlockFromBin(FreeBlock *fBlock)

}

void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz)

}

void AllLargeBlocksList::add(LargeMemoryBlock *lmb)

}

void AllLargeBlocksList::remove(LargeMemoryBlock *lmb)

}

void Backend::putLargeBlock(LargeMemoryBlock *lmb)

}