#include "Python.h"

#ifdef WITH_PYMALLOC

#ifdef WITH_VALGRIND

#include <valgrind/valgrind.h>

#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)

# define UNLIKELY(value) __builtin_expect((value), 0)

# define UNLIKELY(value) (value)

static int running_on_valgrind = -1;

/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */

#define ALIGNMENT 8 /* must be 2^N */

#define ALIGNMENT_SHIFT 3

#define ALIGNMENT_MASK (ALIGNMENT - 1)

#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)

#define SMALL_REQUEST_THRESHOLD 256

#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT)

#define SYSTEM_PAGE_SIZE (4 * 1024)

#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1)

#ifdef WITH_MEMORY_LIMITS

#ifndef SMALL_MEMORY_LIMIT

#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */

#define ARENA_SIZE (256 << 10) /* 256KB */

#ifdef WITH_MEMORY_LIMITS

#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE)

#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */

#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK

#define SIMPLELOCK_DECL(lock) /* simple lock declaration */

#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */

#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */

#define SIMPLELOCK_LOCK(lock) /* acquire released lock */

#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */

#undef uchar

#define uchar unsigned char /* assuming == 8 bits */

#undef uint

#define uint unsigned int /* assuming >= 16 bits */

#undef ulong

#define ulong unsigned long /* assuming >= 32 bits */

#undef uptr

#define uptr Py_uintptr_t

typedef uchar block;

struct pool_header {

};

typedef struct pool_header *poolp;

struct arena_object {

};

#undef ROUNDUP

#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK)

#define POOL_OVERHEAD ROUNDUP(sizeof(struct pool_header))

#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */

#define POOL_ADDR(P) ((poolp)((uptr)(P) & ~(uptr)POOL_SIZE_MASK))

#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I))

SIMPLELOCK_DECL(_malloc_lock)

#define LOCK() SIMPLELOCK_LOCK(_malloc_lock)

#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock)

#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock)

#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock)

#define PTA(x) ((poolp )((uchar *)&(usedpools[2*(x)]) - 2*sizeof(block *)))

#define PT(x) PTA(x), PTA(x)

static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = {

PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7)

#if NB_SMALL_SIZE_CLASSES > 8

, PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15)

#if NB_SMALL_SIZE_CLASSES > 16

, PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23)

#if NB_SMALL_SIZE_CLASSES > 24

, PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31)

#if NB_SMALL_SIZE_CLASSES > 32

, PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39)

#if NB_SMALL_SIZE_CLASSES > 40

, PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47)

#if NB_SMALL_SIZE_CLASSES > 48

, PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55)

#if NB_SMALL_SIZE_CLASSES > 56

, PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63)

#endif /* NB_SMALL_SIZE_CLASSES > 56 */

#endif /* NB_SMALL_SIZE_CLASSES > 48 */

#endif /* NB_SMALL_SIZE_CLASSES > 40 */

#endif /* NB_SMALL_SIZE_CLASSES > 32 */

#endif /* NB_SMALL_SIZE_CLASSES > 24 */

#endif /* NB_SMALL_SIZE_CLASSES > 16 */

#endif /* NB_SMALL_SIZE_CLASSES > 8 */

};

static struct arena_object* arenas = NULL;

static uint maxarenas = 0;

static struct arena_object* unused_arena_objects = NULL;

static struct arena_object* usable_arenas = NULL;

#define INITIAL_ARENA_OBJECTS 16

static size_t narenas_currently_allocated = 0;

#ifdef PYMALLOC_DEBUG

static size_t ntimes_arena_allocated = 0;

static size_t narenas_highwater = 0;

static struct arena_object*

new_arena(void)

{

struct arena_object* arenaobj;

uint excess; /* number of bytes above pool alignment */

#ifdef PYMALLOC_DEBUG

if (Py_GETENV("PYTHONMALLOCSTATS"))

_PyObject_DebugMallocStats();

if (unused_arena_objects == NULL) {

uint i;

uint numarenas;

size_t nbytes;

/* Double the number of arena objects on each allocation.

numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;

if (numarenas <= maxarenas)

return NULL; /* overflow */

#if SIZEOF_SIZE_T <= SIZEOF_INT

if (numarenas > PY_SIZE_MAX / sizeof(*arenas))

return NULL; /* overflow */

nbytes = numarenas * sizeof(*arenas);

arenaobj = (struct arena_object *)realloc(arenas, nbytes);

if (arenaobj == NULL)

return NULL;

arenas = arenaobj;

/* We might need to fix pointers that were copied. However,

assert(usable_arenas == NULL);

assert(unused_arena_objects == NULL);

/* Put the new arenas on the unused_arena_objects list. */

for (i = maxarenas; i < numarenas; ++i) {

arenas[i].address = 0; /* mark as unassociated */

arenas[i].nextarena = i < numarenas - 1 ?

&arenas[i+1] : NULL;

}

/* Update globals. */

unused_arena_objects = &arenas[maxarenas];

maxarenas = numarenas;

}

/* Take the next available arena object off the head of the list. */

assert(unused_arena_objects != NULL);

arenaobj = unused_arena_objects;

unused_arena_objects = arenaobj->nextarena;

assert(arenaobj->address == 0);

arenaobj->address = (uptr)malloc(ARENA_SIZE);

if (arenaobj->address == 0) {

/* The allocation failed: return NULL after putting the

arenaobj->nextarena = unused_arena_objects;

unused_arena_objects = arenaobj;

return NULL;

}

++narenas_currently_allocated;

#ifdef PYMALLOC_DEBUG

++ntimes_arena_allocated;

if (narenas_currently_allocated > narenas_highwater)

narenas_highwater = narenas_currently_allocated;

arenaobj->freepools = NULL;

/* pool_address <- first pool-aligned address in the arena

arenaobj->pool_address = (block*)arenaobj->address;

arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE;

assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE);

excess = (uint)(arenaobj->address & POOL_SIZE_MASK);

if (excess != 0) {

--arenaobj->nfreepools;

arenaobj->pool_address += POOL_SIZE - excess;

}

arenaobj->ntotalpools = arenaobj->nfreepools;

return arenaobj;

}

#define Py_ADDRESS_IN_RANGE(P, POOL) \

((arenaindex_temp = (POOL)->arenaindex) < maxarenas && \

(uptr)(P) - arenas[arenaindex_temp].address < (uptr)ARENA_SIZE && \

arenas[arenaindex_temp].address != 0)

#ifdef Py_USING_MEMORY_DEBUGGER

#undef Py_ADDRESS_IN_RANGE

#if defined(__GNUC__) && ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) || \

(__GNUC__ >= 4))

#define Py_NO_INLINE __attribute__((__noinline__))

#define Py_NO_INLINE

int Py_ADDRESS_IN_RANGE(void *P, poolp pool) Py_NO_INLINE;

#undef Py_NO_INLINE

#undef PyObject_Malloc

void *

PyObject_Malloc(size_t nbytes)

{

block *bp;

poolp pool;

poolp next;

uint size;

#ifdef WITH_VALGRIND

if (UNLIKELY(running_on_valgrind == -1))

running_on_valgrind = RUNNING_ON_VALGRIND;

if (UNLIKELY(running_on_valgrind))

goto redirect;

/*

if (nbytes > PY_SSIZE_T_MAX)

return NULL;

/*

if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {

LOCK();

/*

size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;

pool = usedpools[size + size];

if (pool != pool->nextpool) {

/*

++pool->ref.count;

bp = pool->freeblock;

assert(bp != NULL);

if ((pool->freeblock = *(block **)bp) != NULL) {

UNLOCK();

return (void *)bp;

}

/*

if (pool->nextoffset <= pool->maxnextoffset) {

/* There is room for another block. */

pool->freeblock = (block*)pool +

pool->nextoffset;

pool->nextoffset += INDEX2SIZE(size);

*(block **)(pool->freeblock) = NULL;

UNLOCK();

return (void *)bp;

}

/* Pool is full, unlink from used pools. */

next = pool->nextpool;

pool = pool->prevpool;

next->prevpool = pool;

pool->nextpool = next;

UNLOCK();

return (void *)bp;

}

/* There isn't a pool of the right size class immediately

if (usable_arenas == NULL) {

/* No arena has a free pool: allocate a new arena. */

#ifdef WITH_MEMORY_LIMITS

if (narenas_currently_allocated >= MAX_ARENAS) {

UNLOCK();

goto redirect;

}

usable_arenas = new_arena();

if (usable_arenas == NULL) {

UNLOCK();

goto redirect;

}

usable_arenas->nextarena =

usable_arenas->prevarena = NULL;

}

assert(usable_arenas->address != 0);

/* Try to get a cached free pool. */

pool = usable_arenas->freepools;

if (pool != NULL) {

/* Unlink from cached pools. */

usable_arenas->freepools = pool->nextpool;

/* This arena already had the smallest nfreepools

--usable_arenas->nfreepools;

if (usable_arenas->nfreepools == 0) {

/* Wholly allocated: remove. */

assert(usable_arenas->freepools == NULL);

assert(usable_arenas->nextarena == NULL ||

usable_arenas->nextarena->prevarena ==

usable_arenas);

usable_arenas = usable_arenas->nextarena;

if (usable_arenas != NULL) {

usable_arenas->prevarena = NULL;

assert(usable_arenas->address != 0);

}

}

else {

/* nfreepools > 0: it must be that freepools

assert(usable_arenas->freepools != NULL ||

usable_arenas->pool_address <=

(block*)usable_arenas->address +

ARENA_SIZE - POOL_SIZE);

}

init_pool:

/* Frontlink to used pools. */

next = usedpools[size + size]; /* == prev */

pool->nextpool = next;

pool->prevpool = next;

next->nextpool = pool;

next->prevpool = pool;

pool->ref.count = 1;

if (pool->szidx == size) {

/* Luckily, this pool last contained blocks

bp = pool->freeblock;

pool->freeblock = *(block **)bp;

UNLOCK();

return (void *)bp;

}

/*

pool->szidx = size;

size = INDEX2SIZE(size);

bp = (block *)pool + POOL_OVERHEAD;

pool->nextoffset = POOL_OVERHEAD + (size << 1);

pool->maxnextoffset = POOL_SIZE - size;

pool->freeblock = bp + size;

*(block **)(pool->freeblock) = NULL;

UNLOCK();

return (void *)bp;

}

/* Carve off a new pool. */

assert(usable_arenas->nfreepools > 0);

assert(usable_arenas->freepools == NULL);

pool = (poolp)usable_arenas->pool_address;

assert((block*)pool <= (block*)usable_arenas->address +

ARENA_SIZE - POOL_SIZE);

pool->arenaindex = usable_arenas - arenas;

assert(&arenas[pool->arenaindex] == usable_arenas);

pool->szidx = DUMMY_SIZE_IDX;

usable_arenas->pool_address += POOL_SIZE;

--usable_arenas->nfreepools;

if (usable_arenas->nfreepools == 0) {

assert(usable_arenas->nextarena == NULL ||

usable_arenas->nextarena->prevarena ==

usable_arenas);

/* Unlink the arena: it is completely allocated. */

usable_arenas = usable_arenas->nextarena;

if (usable_arenas != NULL) {

usable_arenas->prevarena = NULL;

assert(usable_arenas->address != 0);

}

}

goto init_pool;

}

/* The small block allocator ends here. */

redirect:

/* Redirect the original request to the underlying (libc) allocator.

if (nbytes == 0)

nbytes = 1;

return (void *)malloc(nbytes);

}

#undef PyObject_Free

PyObject_Free(void *p)

{

poolp pool;

block *lastfree;

poolp next, prev;

uint size;

#ifndef Py_USING_MEMORY_DEBUGGER

uint arenaindex_temp;

if (p == NULL) /* free(NULL) has no effect */

return;

#ifdef WITH_VALGRIND

if (UNLIKELY(running_on_valgrind > 0))

goto redirect;

pool = POOL_ADDR(p);

if (Py_ADDRESS_IN_RANGE(p, pool)) {

/* We allocated this address. */

LOCK();

/* Link p to the start of the pool's freeblock list. Since

assert(pool->ref.count > 0); /* else it was empty */

*(block **)p = lastfree = pool->freeblock;

pool->freeblock = (block *)p;

if (lastfree) {

struct arena_object* ao;

uint nf; /* ao->nfreepools */

/* freeblock wasn't NULL, so the pool wasn't full,

if (--pool->ref.count != 0) {

/* pool isn't empty: leave it in usedpools */

UNLOCK();

return;

}

/* Pool is now empty: unlink from usedpools, and

next = pool->nextpool;

prev = pool->prevpool;

next->prevpool = prev;