#include <string.h> /* for memset */

#include <errno.h>

#include "tbbmalloc_internal.h"

namespace rml {

namespace internal {

#if USE_DEFAULT_MEMORY_MAPPING

#include "MapMemory.h"

#else

#endif

void* getRawMemory (size_t size, PageType pageType) {

return MapMemory(size, pageType);

}

int freeRawMemory (void *object, size_t size) {

return UnmapMemory(object, size);

}

#if CHECK_ALLOCATION_RANGE

void Backend::UsedAddressRange::registerAlloc(uintptr_t left, uintptr_t right)

{

MallocMutex::scoped_lock lock(mutex);

if (left < leftBound.load(std::memory_order_relaxed))

leftBound.store(left, std::memory_order_relaxed);

if (right > rightBound.load(std::memory_order_relaxed))

rightBound.store(right, std::memory_order_relaxed);

MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed), ASSERT_TEXT);

MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);

MALLOC_ASSERT(leftBound.load(std::memory_order_relaxed) <= left && right <= rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);

}

void Backend::UsedAddressRange::registerFree(uintptr_t left, uintptr_t right)

{

MallocMutex::scoped_lock lock(mutex);

if (leftBound.load(std::memory_order_relaxed) == left) {

if (rightBound.load(std::memory_order_relaxed) == right) {

leftBound.store(ADDRESS_UPPER_BOUND, std::memory_order_relaxed);

rightBound.store(0, std::memory_order_relaxed);

} else

leftBound.store(right, std::memory_order_relaxed);

} else if (rightBound.load(std::memory_order_relaxed) == right)

rightBound.store(left, std::memory_order_relaxed);

MALLOC_ASSERT((!rightBound.load(std::memory_order_relaxed) && leftBound.load(std::memory_order_relaxed) == ADDRESS_UPPER_BOUND)

|| leftBound.load(std::memory_order_relaxed) < rightBound.load(std::memory_order_relaxed), ASSERT_TEXT);

}

#endif // CHECK_ALLOCATION_RANGE

extern HugePagesStatus hugePages;

void *Backend::allocRawMem(size_t &size)

{

void *res = nullptr;

size_t allocSize = 0;

if (extMemPool->userPool()) {

if (extMemPool->fixedPool && bootsrapMemDone == bootsrapMemStatus.load(std::memory_order_acquire))

return nullptr;

MALLOC_ASSERT(bootsrapMemStatus != bootsrapMemNotDone,

"Backend::allocRawMem() called prematurely?");

// TODO: support for raw mem not aligned at sizeof(uintptr_t)

// memory from fixed pool is asked once and only once

allocSize = alignUpGeneric(size, extMemPool->granularity);

res = (*extMemPool->rawAlloc)(extMemPool->poolId, allocSize);

} else {

// Align allocation on page size

size_t pageSize = hugePages.isEnabled ? hugePages.getGranularity() : extMemPool->granularity;

MALLOC_ASSERT(pageSize, "Page size cannot be zero.");

allocSize = alignUpGeneric(size, pageSize);

// If user requested huge pages and they are available, try to use preallocated ones firstly.

// If there are none, lets check transparent huge pages support and use them instead.

if (hugePages.isEnabled) {

if (hugePages.isHPAvailable) {

res = getRawMemory(allocSize, PREALLOCATED_HUGE_PAGE);

}

if (!res && hugePages.isTHPAvailable) {

res = getRawMemory(allocSize, TRANSPARENT_HUGE_PAGE);

}

}

if (!res) {

res = getRawMemory(allocSize, REGULAR);

}

}

if (res) {

MALLOC_ASSERT(allocSize > 0, "Invalid size of an allocated region.");

size = allocSize;

if (!extMemPool->userPool())

usedAddrRange.registerAlloc((uintptr_t)res, (uintptr_t)res+size);

#if MALLOC_DEBUG

volatile size_t curTotalSize = totalMemSize; // to read global value once

MALLOC_ASSERT(curTotalSize+size > curTotalSize, "Overflow allocation size.");

#endif

totalMemSize.fetch_add(size);

}

return res;

}

bool Backend::freeRawMem(void *object, size_t size)

}

class GuardedSize : tbb::detail::no_copy {

};

struct MemRegion {

};

class BlockMutexes {

};

class FreeBlock : BlockMutexes {

};

struct LastFreeBlock : public FreeBlock {

MemRegion *memRegion;

};

const size_t FreeBlock::minBlockSize = sizeof(FreeBlock);

inline bool BackendSync::waitTillBlockReleased(intptr_t startModifiedCnt)

}

void CoalRequestQ::putBlock(FreeBlock *fBlock)

}

FreeBlock *CoalRequestQ::getAll()

{

for (;;) {

FreeBlock *myBlToFree = blocksToFree.load(std::memory_order_acquire);

if (!myBlToFree) {

return nullptr;

} else {

if (blocksToFree.compare_exchange_strong(myBlToFree, nullptr)) {

return myBlToFree;

} else {

continue;

}

}

}

}

inline void CoalRequestQ::blockWasProcessed()

}

FreeBlock *Backend::IndexedBins::getFromBin(int binIdx, BackendSync *sync, size_t size,

bool needAlignedRes, bool alignedBin, bool wait, int *binLocked)

{

Bin *b = &freeBins[binIdx];

try_next:

FreeBlock *fBlock = nullptr;

if (!b->empty()) {

bool locked = false;

MallocMutex::scoped_lock scopedLock(b->tLock, wait, &locked);

if (!locked) {

if (binLocked) (*binLocked)++;

return nullptr;

}

for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; curr = curr->next) {

size_t szBlock = curr->tryLockBlock();

if (!szBlock) {

// block is locked, re-do bin lock, as there is no place to spin

// while block coalescing

goto try_next;

}

// GENERAL CASE

if (alignedBin || !needAlignedRes) {

size_t splitSz = szBlock - size;

// If we got a block as split result, it must have a room for control structures.

if (szBlock >= size && (splitSz >= FreeBlock::minBlockSize || !splitSz))

fBlock = curr;

} else {

// SPECIAL CASE, to get aligned block from unaligned bin we have to cut the middle of a block

// and return remaining left and right part. Possible only in fixed pool scenario, assert for this

// is set inside splitBlock() function.

void *newB = alignUp(curr, slabSize);

uintptr_t rightNew = (uintptr_t)newB + size;

uintptr_t rightCurr = (uintptr_t)curr + szBlock;

// Check if the block size is sufficient,

// and also left and right split results are either big enough or non-existent

if (rightNew <= rightCurr

&& (newB == curr || ((uintptr_t)newB - (uintptr_t)curr) >= FreeBlock::minBlockSize)

&& (rightNew == rightCurr || (rightCurr - rightNew) >= FreeBlock::minBlockSize))

fBlock = curr;

}

if (fBlock) {

// consume must be called before result of removing from a bin is visible externally.

sync->blockConsumed();

// TODO: think about cases when block stays in the same bin

b->removeBlock(fBlock);

if (freeBins[binIdx].empty())

bitMask.set(binIdx, false);

fBlock->sizeTmp = szBlock;

break;

} else { // block size is not valid, search for next block in the bin

curr->setMeFree(szBlock);

curr->rightNeig(szBlock)->setLeftFree(szBlock);

}

}

}

return fBlock;

}

bool Backend::IndexedBins::tryReleaseRegions(int binIdx, Backend *backend)

{

Bin *b = &freeBins[binIdx];

FreeBlock *fBlockList = nullptr;

// got all blocks from the bin and re-do coalesce on them

// to release single-block regions

try_next:

if (!b->empty()) {

MallocMutex::scoped_lock binLock(b->tLock);

for (FreeBlock *curr = b->head.load(std::memory_order_relaxed); curr; ) {

size_t szBlock = curr->tryLockBlock();

if (!szBlock)

goto try_next;

FreeBlock *next = curr->next;

b->removeBlock(curr);

curr->sizeTmp = szBlock;

curr->nextToFree = fBlockList;

fBlockList = curr;

curr = next;

}

}

return backend->coalescAndPutList(fBlockList, /*forceCoalescQDrop=*/true,

/*reportBlocksProcessed=*/false);

}

void Backend::Bin::removeBlock(FreeBlock *fBlock)

{

MALLOC_ASSERT(fBlock->next||fBlock->prev||fBlock== head.load(std::memory_order_relaxed),

"Detected that a block is not in the bin.");

if (head.load(std::memory_order_relaxed) == fBlock)

head.store(fBlock->next, std::memory_order_relaxed);

if (tail == fBlock)

tail = fBlock->prev;

if (fBlock->prev)

fBlock->prev->next = fBlock->next;

if (fBlock->next)

fBlock->next->prev = fBlock->prev;

}

void Backend::IndexedBins::addBlock(int binIdx, FreeBlock *fBlock, size_t /* blockSz */, bool addToTail)

{

Bin *b = &freeBins[binIdx];

fBlock->myBin = binIdx;

fBlock->next = fBlock->prev = nullptr;

{

MallocMutex::scoped_lock scopedLock(b->tLock);

if (addToTail) {

fBlock->prev = b->tail;

b->tail = fBlock;

if (fBlock->prev)

fBlock->prev->next = fBlock;

if (!b->head.load(std::memory_order_relaxed))

b->head.store(fBlock, std::memory_order_relaxed);

} else {

fBlock->next = b->head.load(std::memory_order_relaxed);

b->head.store(fBlock, std::memory_order_relaxed);

if (fBlock->next)

fBlock->next->prev = fBlock;

if (!b->tail)

b->tail = fBlock;

}

}

bitMask.set(binIdx, true);

}

bool Backend::IndexedBins::tryAddBlock(int binIdx, FreeBlock *fBlock, bool addToTail)

{

bool locked = false;

Bin *b = &freeBins[binIdx];

fBlock->myBin = binIdx;

if (addToTail) {

fBlock->next = nullptr;

{

MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);

if (!locked)

return false;

fBlock->prev = b->tail;

b->tail = fBlock;

if (fBlock->prev)

fBlock->prev->next = fBlock;

if (!b->head.load(std::memory_order_relaxed))

b->head.store(fBlock, std::memory_order_relaxed);

}

} else {

fBlock->prev = nullptr;

{

MallocMutex::scoped_lock scopedLock(b->tLock, /*wait=*/false, &locked);

if (!locked)

return false;

fBlock->next = b->head.load(std::memory_order_relaxed);

b->head.store(fBlock, std::memory_order_relaxed);

if (fBlock->next)

fBlock->next->prev = fBlock;

if (!b->tail)

b->tail = fBlock;

}

}

bitMask.set(binIdx, true);

return true;

}

void Backend::IndexedBins::reset()

{

for (unsigned i=0; i<Backend::freeBinsNum; i++)

freeBins[i].reset();

bitMask.reset();

}

void Backend::IndexedBins::lockRemoveBlock(int binIdx, FreeBlock *fBlock)

{

MallocMutex::scoped_lock scopedLock(freeBins[binIdx].tLock);

freeBins[binIdx].removeBlock(fBlock);

if (freeBins[binIdx].empty())

bitMask.set(binIdx, false);

}

bool ExtMemoryPool::regionsAreReleaseable() const

{

return !keepAllMemory && !delayRegsReleasing;

}

FreeBlock *Backend::splitBlock(FreeBlock *fBlock, int num, size_t size, bool blockIsAligned, bool needAlignedBlock)

{

const size_t totalSize = num * size;

// SPECIAL CASE, for unaligned block we have to cut the middle of a block

// and return remaining left and right part. Possible only in a fixed pool scenario.

if (needAlignedBlock && !blockIsAligned) {

MALLOC_ASSERT(extMemPool->fixedPool,

"Aligned block request from unaligned bin possible only in fixed pool scenario.");

// Space to use is in the middle

FreeBlock *newBlock = alignUp(fBlock, slabSize);

FreeBlock *rightPart = (FreeBlock*)((uintptr_t)newBlock + totalSize);

uintptr_t fBlockEnd = (uintptr_t)fBlock + fBlock->sizeTmp;

// Return free right part

if ((uintptr_t)rightPart != fBlockEnd) {

rightPart->initHeader(); // to prevent coalescing rightPart with fBlock

size_t rightSize = fBlockEnd - (uintptr_t)rightPart;

coalescAndPut(rightPart, rightSize, toAlignedBin(rightPart, rightSize));

}

// And free left part

if (newBlock != fBlock) {

newBlock->initHeader(); // to prevent coalescing fBlock with newB

size_t leftSize = (uintptr_t)newBlock - (uintptr_t)fBlock;

coalescAndPut(fBlock, leftSize, toAlignedBin(fBlock, leftSize));

}

fBlock = newBlock;

} else if (size_t splitSize = fBlock->sizeTmp - totalSize) { // need to split the block

// GENERAL CASE, cut the left or right part of the block

FreeBlock *splitBlock = nullptr;

if (needAlignedBlock) {

// For slab aligned blocks cut the right side of the block

// and return it to a requester, original block returns to backend

splitBlock = fBlock;

fBlock = (FreeBlock*)((uintptr_t)splitBlock + splitSize);

fBlock->initHeader();

} else {

// For large object blocks cut original block and put free right part to backend

splitBlock = (FreeBlock*)((uintptr_t)fBlock + totalSize);

splitBlock->initHeader();

}

// Mark free block as it`s parent only when the requested type (needAlignedBlock)

// and returned from Bins/OS block (isAligned) are equal (XOR operation used)

bool markAligned = (blockIsAligned ^ needAlignedBlock) ? toAlignedBin(splitBlock, splitSize) : blockIsAligned;

coalescAndPut(splitBlock, splitSize, markAligned);

}

MALLOC_ASSERT(!needAlignedBlock || isAligned(fBlock, slabSize), "Expect to get aligned block, if one was requested.");

FreeBlock::markBlocks(fBlock, num, size);

return fBlock;

}

size_t Backend::getMaxBinnedSize() const

}

inline bool Backend::MaxRequestComparator::operator()(size_t oldMaxReq, size_t requestSize) const

{

return requestSize > oldMaxReq && requestSize < backend->getMaxBinnedSize();

}

FreeBlock *Backend::releaseMemInCaches(intptr_t startModifiedCnt,

int *lockedBinsThreshold, int numOfLockedBins)

{

// something released from caches

if (extMemPool->hardCachesCleanup(false))

return (FreeBlock*)VALID_BLOCK_IN_BIN;

if (bkndSync.waitTillBlockReleased(startModifiedCnt))

return (FreeBlock*)VALID_BLOCK_IN_BIN;

// OS can't give us more memory, but we have some in locked bins

if (*lockedBinsThreshold && numOfLockedBins) {

*lockedBinsThreshold = 0;

return (FreeBlock*)VALID_BLOCK_IN_BIN;

}

return nullptr; // nothing found, give up

}

FreeBlock *Backend::askMemFromOS(size_t blockSize, intptr_t startModifiedCnt,

int *lockedBinsThreshold, int numOfLockedBins,

bool *splittableRet, bool needSlabRegion)

{

FreeBlock *block;

// The block sizes can be divided into 3 groups:

// 1. "quite small": popular object size, we are in bootstarp or something

// like; request several regions.

// 2. "quite large": we want to have several such blocks in the region

// but not want several pre-allocated regions.

// 3. "huge": exact fit, we allocate only one block and do not allow

// any other allocations to placed in a region.

// Dividing the block sizes in these groups we are trying to balance between

// too small regions (that leads to fragmentation) and too large ones (that

// leads to excessive address space consumption). If a region is "too

// large", allocate only one, to prevent fragmentation. It supposedly

// doesn't hurt performance, because the object requested by user is large.

// Bounds for the groups are:

const size_t maxBinned = getMaxBinnedSize();

const size_t quiteSmall = maxBinned / 8;

const size_t quiteLarge = maxBinned;

if (blockSize >= quiteLarge) {

// Do not interact with other threads via semaphores, as for exact fit

// we can't share regions with them, memory requesting is individual.

block = addNewRegion(blockSize, MEMREG_ONE_BLOCK, /*addToBin=*/false);

if (!block)

return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins);

*splittableRet = false;

} else {

const size_t regSz_sizeBased = alignUp(4*maxRequestedSize, 1024*1024);

// Another thread is modifying backend while we can't get the block.

// Wait while it leaves and re-do the scan

// before trying other ways to extend the backend.

if (bkndSync.waitTillBlockReleased(startModifiedCnt)

// semaphore is protecting adding more more memory from OS

|| memExtendingSema.wait())

return (FreeBlock*)VALID_BLOCK_IN_BIN;

if (startModifiedCnt != bkndSync.getNumOfMods()) {

memExtendingSema.signal();

return (FreeBlock*)VALID_BLOCK_IN_BIN;

}

if (blockSize < quiteSmall) {

// For this size of blocks, add NUM_OF_REG "advance" regions in bin,

// and return one as a result.

// TODO: add to bin first, because other threads can use them right away.

// This must be done carefully, because blocks in bins can be released

// in releaseCachesToLimit().

const unsigned NUM_OF_REG = 3;

MemRegionType regType = needSlabRegion ? MEMREG_SLAB_BLOCKS : MEMREG_LARGE_BLOCKS;

block = addNewRegion(regSz_sizeBased, regType, /*addToBin=*/false);

if (block)

for (unsigned idx=0; idx<NUM_OF_REG; idx++)

if (! addNewRegion(regSz_sizeBased, regType, /*addToBin=*/true))

break;

} else {

block = addNewRegion(regSz_sizeBased, MEMREG_LARGE_BLOCKS, /*addToBin=*/false);

}

memExtendingSema.signal();

// no regions found, try to clean cache

if (!block || block == (FreeBlock*)VALID_BLOCK_IN_BIN)

return releaseMemInCaches(startModifiedCnt, lockedBinsThreshold, numOfLockedBins);

// Since a region can hold more than one block it can be split.

*splittableRet = true;

}

// after asking memory from OS, release caches if we above the memory limits

releaseCachesToLimit();

return block;

}

void Backend::releaseCachesToLimit()

}

int Backend::IndexedBins::getMinNonemptyBin(unsigned startBin) const

{

int p = bitMask.getMinTrue(startBin);

return p == -1 ? Backend::freeBinsNum : p;

}

FreeBlock *Backend::IndexedBins::findBlock(int nativeBin, BackendSync *sync, size_t size,

bool needAlignedBlock, bool alignedBin, int *numOfLockedBins)

{

for (int i=getMinNonemptyBin(nativeBin); i<(int)freeBinsNum; i=getMinNonemptyBin(i+1))

if (FreeBlock *block = getFromBin(i, sync, size, needAlignedBlock, alignedBin, /*wait=*/false, numOfLockedBins))

return block;

return nullptr;

}

void Backend::requestBootstrapMem()

}

FreeBlock *Backend::genericGetBlock(int num, size_t size, bool needAlignedBlock)

{

FreeBlock *block = nullptr;

const size_t totalReqSize = num*size;

// no splitting after requesting new region, asks exact size

const int nativeBin = sizeToBin(totalReqSize);

requestBootstrapMem();

// If we found 2 or less locked bins, it's time to ask more memory from OS.

// But nothing can be asked from fixed pool. And we prefer wait, not ask

// for more memory, if block is quite large.

int lockedBinsThreshold = extMemPool->fixedPool || size>=maxBinned_SmallPage? 0 : 2;

// Find maximal requested size limited by getMaxBinnedSize()

AtomicUpdate(maxRequestedSize, totalReqSize, MaxRequestComparator(this));

scanCoalescQ(/*forceCoalescQDrop=*/false);

bool splittable = true;

for (;;) {

const intptr_t startModifiedCnt = bkndSync.getNumOfMods();

int numOfLockedBins;

intptr_t cleanCnt;

do {

cleanCnt = backendCleanCnt.load(std::memory_order_acquire);

numOfLockedBins = 0;

if (needAlignedBlock) {

block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,

/*alignedBin=*/true, &numOfLockedBins);

if (!block && extMemPool->fixedPool)

block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,

/*alignedBin=*/false, &numOfLockedBins);

} else {

block = freeLargeBlockBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,

/*alignedBin=*/false, &numOfLockedBins);

if (!block && extMemPool->fixedPool)

block = freeSlabAlignedBins.findBlock(nativeBin, &bkndSync, num*size, needAlignedBlock,

/*alignedBin=*/true, &numOfLockedBins);

}

} while (!block && (numOfLockedBins>lockedBinsThreshold || cleanCnt % 2 == 1 ||

cleanCnt != backendCleanCnt.load(std::memory_order_acquire)));

if (block)

break;

bool retScanCoalescQ = scanCoalescQ(/*forceCoalescQDrop=*/true);

bool retSoftCachesCleanup = extMemPool->softCachesCleanup();

if (!(retScanCoalescQ || retSoftCachesCleanup)) {

// bins are not updated,

// only remaining possibility is to ask for more memory

block = askMemFromOS(totalReqSize, startModifiedCnt, &lockedBinsThreshold,

numOfLockedBins, &splittable, needAlignedBlock);

if (!block)

return nullptr;

if (block != (FreeBlock*)VALID_BLOCK_IN_BIN) {

// size can be increased in askMemFromOS, that's why >=

MALLOC_ASSERT(block->sizeTmp >= size, ASSERT_TEXT);

break;

}

// valid block somewhere in bins, let's find it

block = nullptr;

}

}

MALLOC_ASSERT(block, ASSERT_TEXT);

if (splittable) {

// At this point we have to be sure that slabAligned attribute describes the right block state

block = splitBlock(block, num, size, block->slabAligned, needAlignedBlock);

}

// matched blockConsumed() from startUseBlock()

bkndSync.blockReleased();

return block;

}

LargeMemoryBlock *Backend::getLargeBlock(size_t size)

{

LargeMemoryBlock *lmb =

(LargeMemoryBlock*)genericGetBlock(1, size, /*needAlignedRes=*/false);

if (lmb) {

lmb->unalignedSize = size;

if (extMemPool->userPool())

extMemPool->lmbList.add(lmb);

}

return lmb;

}

BlockI *Backend::getSlabBlock(int num) {

BlockI *b = (BlockI*)genericGetBlock(num, slabSize, /*slabAligned=*/true);

MALLOC_ASSERT(isAligned(b, slabSize), ASSERT_TEXT);

return b;

}

void Backend::putSlabBlock(BlockI *block) {

genericPutBlock((FreeBlock *)block, slabSize, /*slabAligned=*/true);

}

void *Backend::getBackRefSpace(size_t size, bool *rawMemUsed)

{

// This block is released only at shutdown, so it can prevent

// a entire region releasing when it's received from the backend,

// so prefer getRawMemory using.

if (void *ret = getRawMemory(size, REGULAR)) {

*rawMemUsed = true;

return ret;

}

void *ret = genericGetBlock(1, size, /*needAlignedRes=*/false);

if (ret) *rawMemUsed = false;

return ret;

}

void Backend::putBackRefSpace(void *b, size_t size, bool rawMemUsed)

}

void Backend::removeBlockFromBin(FreeBlock *fBlock)

}

void Backend::genericPutBlock(FreeBlock *fBlock, size_t blockSz, bool slabAligned)

}

void AllLargeBlocksList::add(LargeMemoryBlock *lmb)

}

void AllLargeBlocksList::remove(LargeMemoryBlock *lmb)

}

void Backend::putLargeBlock(LargeMemoryBlock *lmb)

}

void Backend::returnLargeObject(LargeMemoryBlock *lmb)

}

#if BACKEND_HAS_MREMAP

void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment)

{

// no remap for user pools and for object too small that living in bins

if (inUserPool() || min(oldSize, newSize)<maxBinned_SmallPage

// during remap, can't guarantee alignment more strict than current or

// more strict than page alignment

|| !isAligned(ptr, alignment) || alignment>extMemPool->granularity)

return nullptr;

const LargeMemoryBlock* lmbOld = ((LargeObjectHdr *)ptr - 1)->memoryBlock;

const size_t oldUnalignedSize = lmbOld->unalignedSize;

FreeBlock *oldFBlock = (FreeBlock *)lmbOld;

FreeBlock *right = oldFBlock->rightNeig(oldUnalignedSize);

// in every region only one block can have LAST_REGION_BLOCK on right,

// so don't need no synchronization

if (!right->isLastRegionBlock())

return nullptr;

MemRegion *oldRegion = static_cast<LastFreeBlock*>(right)->memRegion;

MALLOC_ASSERT( oldRegion < ptr, ASSERT_TEXT );

const size_t oldRegionSize = oldRegion->allocSz;

if (oldRegion->type != MEMREG_ONE_BLOCK)

return nullptr; // we are not single in the region

const size_t userOffset = (uintptr_t)ptr - (uintptr_t)oldRegion;

const size_t alignedSize = LargeObjectCache::alignToBin(newSize + userOffset);

const size_t requestSize =

alignUp(sizeof(MemRegion) + alignedSize + sizeof(LastFreeBlock), extMemPool->granularity);

if (requestSize < alignedSize) // is wrapped around?

return nullptr;

regionList.remove(oldRegion);

// The deallocation should be registered in address range before mremap to

// prevent a race condition with allocation on another thread.

// (OS can reuse the memory and registerAlloc will be missed on another thread)

usedAddrRange.registerFree((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize);

void *ret = mremap(oldRegion, oldRegion->allocSz, requestSize, MREMAP_MAYMOVE);

if (MAP_FAILED == ret) { // can't remap, revert and leave

regionList.add(oldRegion);

usedAddrRange.registerAlloc((uintptr_t)oldRegion, (uintptr_t)oldRegion + oldRegionSize);

return nullptr;

}

MemRegion *region = (MemRegion*)ret;

MALLOC_ASSERT(region->type == MEMREG_ONE_BLOCK, ASSERT_TEXT);