#if !__OBJC2__

#include "objc-private.h"

#include "objc-cache-old.h"

#include "hashtable2.h"

typedef struct {

SEL name; // same layout as struct old_method

void *unused;

IMP imp; // same layout as struct old_method

} cache_entry;

static const int _class_slow_grow = 1;

enum {

INIT_CACHE_SIZE_LOG2 = 2,

INIT_CACHE_SIZE = (1 << INIT_CACHE_SIZE_LOG2)

};

#define TABLE_SIZE(count) ((count - 1) * sizeof(cache_entry *))

#if !TARGET_OS_WIN32

# define CACHE_ALLOCATOR

#endif

#define CACHE_ALLOCATOR_MIN 512

#define CACHE_QUANTUM (CACHE_ALLOCATOR_MIN+sizeof(struct objc_cache)-sizeof(cache_entry*))

#define CACHE_REGION_SIZE ((128*1024 / CACHE_QUANTUM) * CACHE_QUANTUM)

static uintptr_t cache_allocator_mask_for_size(size_t size)

{

return (size - sizeof(struct objc_cache)) / sizeof(cache_entry *);

}

static size_t cache_allocator_size_for_mask(uintptr_t mask)

{

size_t requested = sizeof(struct objc_cache) + TABLE_SIZE(mask+1);

size_t actual = CACHE_QUANTUM;

while (actual < requested) actual += CACHE_QUANTUM;

return actual;

}

#ifdef OBJC_INSTRUMENTED

typedef struct

{

unsigned int hitCount; // cache lookup success tally

unsigned int hitProbes; // sum entries checked to hit

unsigned int maxHitProbes; // max entries checked to hit

unsigned int missCount; // cache lookup no-find tally

unsigned int missProbes; // sum entries checked to miss

unsigned int maxMissProbes; // max entries checked to miss

unsigned int flushCount; // cache flush tally

unsigned int flushedEntries; // sum cache entries flushed

unsigned int maxFlushedEntries; // max cache entries flushed

} CacheInstrumentation;

#define CACHE_INSTRUMENTATION(cache) (CacheInstrumentation *) &cache->buckets[cache->mask + 1];

#endif

static int totalCacheFills = 0;

#ifdef OBJC_INSTRUMENTED

unsigned int LinearFlushCachesCount = 0;

unsigned int LinearFlushCachesVisitedCount = 0;

unsigned int MaxLinearFlushCachesVisitedCount = 0;

unsigned int NonlinearFlushCachesCount = 0;

unsigned int NonlinearFlushCachesClassCount = 0;

unsigned int NonlinearFlushCachesVisitedCount = 0;

unsigned int MaxNonlinearFlushCachesVisitedCount = 0;

unsigned int IdealFlushCachesCount = 0;

unsigned int MaxIdealFlushCachesCount = 0;

#endif

struct objc_cache _objc_empty_cache =

{

0, // mask

0, // occupied

{ NULL } // buckets

};

#ifdef OBJC_INSTRUMENTED

CacheInstrumentation emptyCacheInstrumentation = {0};

#endif

static bool _cache_isEmpty(Cache cache);

static Cache _cache_malloc(uintptr_t slotCount);

static Cache _cache_create(Class cls);

static Cache _cache_expand(Class cls);

static int _collecting_in_critical(void);

static void _garbage_make_room(void);

static void _cache_collect_free(void *data, size_t size);

#if defined(CACHE_ALLOCATOR)

static bool cache_allocator_is_block(void *block);

static Cache cache_allocator_calloc(size_t size);

static void cache_allocator_free(void *block);

#endif

static unsigned int cache_counts[16];

static size_t cache_allocations;

static size_t cache_collections;

static size_t cache_allocator_regions;

static size_t log2u(size_t x)

{

unsigned int log;

log = 0;

while (x >>= 1)

log += 1;

return log;

}

static bool _cache_isEmpty(Cache cache)

{

return (cache == NULL || cache == (Cache)&_objc_empty_cache || cache->mask == 0);

}

static Cache _cache_malloc(uintptr_t slotCount)

{

Cache new_cache;

size_t size;

cacheUpdateLock.assertLocked();

// Allocate table (why not check for failure?)

size = sizeof(struct objc_cache) + TABLE_SIZE(slotCount);

#if defined(OBJC_INSTRUMENTED)

// Custom cache allocator can't handle instrumentation.

size += sizeof(CacheInstrumentation);

new_cache = calloc(size, 1);

new_cache->mask = slotCount - 1;

#elif !defined(CACHE_ALLOCATOR)

// fixme cache allocator implementation isn't 64-bit clean

new_cache = calloc(size, 1);

new_cache->mask = (unsigned int)(slotCount - 1);

#else

if (size < CACHE_ALLOCATOR_MIN) {

new_cache = (Cache)calloc(size, 1);

new_cache->mask = slotCount - 1;

// occupied and buckets and instrumentation are all zero

} else {

new_cache = cache_allocator_calloc(size);

// mask is already set

// occupied and buckets and instrumentation are all zero

}

#endif

if (PrintCaches) {

size_t bucket = log2u(slotCount);

if (bucket < sizeof(cache_counts) / sizeof(cache_counts[0])) {

cache_counts[bucket]++;

}

cache_allocations++;

}

return new_cache;

}

static inline int isPowerOf2(unsigned long l) { return 1 == __builtin_popcountl(l); }

static void _cache_free_block(void *block)

{

cacheUpdateLock.assertLocked();

#if !TARGET_OS_WIN32

if (PrintCaches) {

Cache cache = (Cache)block;

size_t slotCount = cache->mask + 1;

if (isPowerOf2(slotCount)) {

size_t bucket = log2u(slotCount);

if (bucket < sizeof(cache_counts) / sizeof(cache_counts[0])) {

cache_counts[bucket]--;

}

}

}

#endif

#if defined(CACHE_ALLOCATOR)

if (cache_allocator_is_block(block)) {

cache_allocator_free(block);

} else

#endif

{

free(block);

}

}

void _cache_free(Cache cache)

{

unsigned int i;

mutex_locker_t lock(cacheUpdateLock);

for (i = 0; i < cache->mask + 1; i++) {

cache_entry *entry = (cache_entry *)cache->buckets[i];

if (entry && entry->imp == _objc_msgForward_impcache) {

_cache_free_block(entry);

}

}

_cache_free_block(cache);

}

static Cache _cache_create(Class cls)

{

Cache new_cache;

cacheUpdateLock.assertLocked();

// Allocate new cache block

new_cache = _cache_malloc(INIT_CACHE_SIZE);

// Install the cache

cls->cache = new_cache;

// Clear the grow flag so that we will re-use the current storage,

// rather than actually grow the cache, when expanding the cache

// for the first time

if (_class_slow_grow) {

cls->setShouldGrowCache(false);

}

// Return our creation

return new_cache;

}

static Cache _cache_expand(Class cls)

{

Cache old_cache;

Cache new_cache;

uintptr_t slotCount;

uintptr_t index;

cacheUpdateLock.assertLocked();

// First growth goes from empty cache to a real one

old_cache = cls->cache;

if (_cache_isEmpty(old_cache))

return _cache_create (cls);

if (_class_slow_grow) {

// Cache grows every other time only.

if (cls->shouldGrowCache()) {

// Grow the cache this time. Don't grow next time.

cls->setShouldGrowCache(false);

}

else {

// Reuse the current cache storage this time. Do grow next time.

cls->setShouldGrowCache(true);

// Clear the valid-entry counter

old_cache->occupied = 0;

// Invalidate all the cache entries

for (index = 0; index < old_cache->mask + 1; index += 1)

{

// Remember what this entry was, so we can possibly

// deallocate it after the bucket has been invalidated

cache_entry *oldEntry = (cache_entry *)old_cache->buckets[index];

// Skip invalid entry

if (!oldEntry)

continue;

// Invalidate this entry

old_cache->buckets[index] = NULL;

// Deallocate "forward::" entry

if (oldEntry->imp == _objc_msgForward_impcache) {

_cache_collect_free (oldEntry, sizeof(cache_entry));

}

}

// Return the same old cache, freshly emptied

return old_cache;

}

}

// Double the cache size

slotCount = (old_cache->mask + 1) << 1;

new_cache = _cache_malloc(slotCount);

#ifdef OBJC_INSTRUMENTED

// Propagate the instrumentation data

{

CacheInstrumentation *oldCacheData;

CacheInstrumentation *newCacheData;

oldCacheData = CACHE_INSTRUMENTATION(old_cache);

newCacheData = CACHE_INSTRUMENTATION(new_cache);

bcopy ((const char *)oldCacheData, (char *)newCacheData, sizeof(CacheInstrumentation));

}

#endif

// Deallocate "forward::" entries from the old cache

for (index = 0; index < old_cache->mask + 1; index++) {

cache_entry *entry = (cache_entry *)old_cache->buckets[index];

if (entry && entry->imp == _objc_msgForward_impcache) {

_cache_collect_free (entry, sizeof(cache_entry));

}

}

// Install new cache

cls->cache = new_cache;

// Deallocate old cache, try freeing all the garbage

_cache_collect_free (old_cache, old_cache->mask * sizeof(cache_entry *));

_cache_collect(false);

return new_cache;

}

bool _cache_fill(Class cls, Method smt, SEL sel)

{

uintptr_t newOccupied;

uintptr_t index;

cache_entry **buckets;

cache_entry *entry;

Cache cache;

cacheUpdateLock.assertUnlocked();

// Never cache before +initialize is done

if (!cls->isInitialized()) {

return NO;

}

// Keep tally of cache additions

totalCacheFills += 1;

mutex_locker_t lock(cacheUpdateLock);

entry = (cache_entry *)smt;

cache = cls->cache;

// Make sure the entry wasn't added to the cache by some other thread

// before we grabbed the cacheUpdateLock.

// Don't use _cache_getMethod() because _cache_getMethod() doesn't

// return forward:: entries.

if (_cache_getImp(cls, sel)) {

return NO; // entry is already cached, didn't add new one

}

// Use the cache as-is if it is less than 3/4 full

newOccupied = cache->occupied + 1;

if ((newOccupied * 4) <= (cache->mask + 1) * 3) {

// Cache is less than 3/4 full.

cache->occupied = (unsigned int)newOccupied;

} else {

// Cache is too full. Expand it.

cache = _cache_expand (cls);

// Account for the addition

cache->occupied += 1;

}

// Scan for the first unused slot and insert there.

// There is guaranteed to be an empty slot because the

// minimum size is 4 and we resized at 3/4 full.

buckets = (cache_entry **)cache->buckets;

for (index = CACHE_HASH(sel, cache->mask);

buckets[index] != NULL;

index = (index+1) & cache->mask)

{

// empty

}

buckets[index] = entry;

return YES; // successfully added new cache entry

}

void _cache_addForwardEntry(Class cls, SEL sel)

{

cache_entry *smt;

smt = (cache_entry *)malloc(sizeof(cache_entry));

smt->name = sel;

smt->imp = _objc_msgForward_impcache;

if (! _cache_fill(cls, (Method)smt, sel)) { // fixme hack

// Entry not added to cache. Don't leak the method struct.

free(smt);

}

}

void _cache_flush(Class cls)

{

Cache cache;

unsigned int index;

cacheUpdateLock.assertLocked();

// Locate cache. Ignore unused cache.

cache = cls->cache;

if (_cache_isEmpty(cache)) return;

#ifdef OBJC_INSTRUMENTED

{

CacheInstrumentation *cacheData;

// Tally this flush

cacheData = CACHE_INSTRUMENTATION(cache);

cacheData->flushCount += 1;

cacheData->flushedEntries += cache->occupied;

if (cache->occupied > cacheData->maxFlushedEntries)

cacheData->maxFlushedEntries = cache->occupied;

}

#endif

// Traverse the cache

for (index = 0; index <= cache->mask; index += 1)

{

// Remember what this entry was, so we can possibly

// deallocate it after the bucket has been invalidated

cache_entry *oldEntry = (cache_entry *)cache->buckets[index];

// Invalidate this entry

cache->buckets[index] = NULL;

// Deallocate "forward::" entry

if (oldEntry && oldEntry->imp == _objc_msgForward_impcache)

_cache_collect_free (oldEntry, sizeof(cache_entry));

}

// Clear the valid-entry counter

cache->occupied = 0;

}

void flush_cache(Class cls)

{

if (cls) {

mutex_locker_t lock(cacheUpdateLock);

_cache_flush(cls);

}

}

#if !TARGET_OS_WIN32

#define PC_SENTINEL 1

#if !__DARWIN_UNIX03

#define __srr0 srr0

#define __eip eip

#endif

static uintptr_t _get_pc_for_thread(thread_t thread)

#if defined(__i386__)

{

i386_thread_state_t state;

unsigned int count = i386_THREAD_STATE_COUNT;

kern_return_t okay = thread_get_state (thread, i386_THREAD_STATE, (thread_state_t)&state, &count);

return (okay == KERN_SUCCESS) ? state.__eip : PC_SENTINEL;

}

#elif defined(__x86_64__)

{

x86_thread_state64_t state;

unsigned int count = x86_THREAD_STATE64_COUNT;

kern_return_t okay = thread_get_state (thread, x86_THREAD_STATE64, (thread_state_t)&state, &count);

return (okay == KERN_SUCCESS) ? state.__rip : PC_SENTINEL;

}

#elif defined(__arm__)

{

arm_thread_state_t state;

unsigned int count = ARM_THREAD_STATE_COUNT;

kern_return_t okay = thread_get_state (thread, ARM_THREAD_STATE, (thread_state_t)&state, &count);

return (okay == KERN_SUCCESS) ? state.__pc : PC_SENTINEL;

}

#else

{

#error _get_pc_for_thread () not implemented for this architecture

}

#endif

#endif

OBJC_EXPORT uintptr_t objc_entryPoints[];

OBJC_EXPORT uintptr_t objc_exitPoints[];

static int _collecting_in_critical(void)

{

#if TARGET_OS_WIN32

return TRUE;

#else

thread_act_port_array_t threads;

unsigned number;

unsigned count;

kern_return_t ret;

int result;

mach_port_t mythread = pthread_mach_thread_np(pthread_self());

// Get a list of all the threads in the current task

ret = task_threads (mach_task_self (), &threads, &number);

if (ret != KERN_SUCCESS)

{

_objc_fatal("task_thread failed (result %d)\n", ret);

}

// Check whether any thread is in the cache lookup code

result = FALSE;

for (count = 0; count < number; count++)

{

int region;

uintptr_t pc;

// Don't bother checking ourselves

if (threads[count] == mythread)

continue;

// Find out where thread is executing

pc = _get_pc_for_thread (threads[count]);

// Check for bad status, and if so, assume the worse (can't collect)

if (pc == PC_SENTINEL)

{

result = TRUE;

goto done;

}

// Check whether it is in the cache lookup code

for (region = 0; objc_entryPoints[region] != 0; region++)

{

if ((pc >= objc_entryPoints[region]) &&

(pc <= objc_exitPoints[region]))

{

result = TRUE;

goto done;

}

}

}

done:

// Deallocate the port rights for the threads

for (count = 0; count < number; count++) {

mach_port_deallocate(mach_task_self (), threads[count]);

}

// Deallocate the thread list

vm_deallocate (mach_task_self (), (vm_address_t) threads, sizeof(threads[0]) * number);

// Return our finding

return result;

#endif

}

static size_t garbage_byte_size = 0;

static size_t garbage_threshold = 1024;

static void **garbage_refs = 0;

static size_t garbage_count = 0;

static size_t garbage_max = 0;

enum {

INIT_GARBAGE_COUNT = 128

};

static void _garbage_make_room(void)

{

static int first = 1;

// Create the collection table the first time it is needed

if (first)

{

first = 0;

garbage_refs = (void**)

malloc(INIT_GARBAGE_COUNT * sizeof(void *));

garbage_max = INIT_GARBAGE_COUNT;

}

// Double the table if it is full

else if (garbage_count == garbage_max)

{

garbage_refs = (void**)

realloc(garbage_refs, garbage_max * 2 * sizeof(void *));

garbage_max *= 2;

}

}

static void _cache_collect_free(void *data, size_t size)

{

cacheUpdateLock.assertLocked();

_garbage_make_room ();

garbage_byte_size += size;

garbage_refs[garbage_count++] = data;

}

void _cache_collect(bool collectALot)

{

cacheUpdateLock.assertLocked();

// Done if the garbage is not full

if (garbage_byte_size < garbage_threshold && !collectALot) {

return;

}

// Synchronize collection with objc_msgSend and other cache readers

if (!collectALot) {

if (_collecting_in_critical ()) {

// objc_msgSend (or other cache reader) is currently looking in

// the cache and might still be using some garbage.

if (PrintCaches) {

_objc_inform ("CACHES: not collecting; "

"objc_msgSend in progress");

}

return;

}

}

else {

// No excuses.

while (_collecting_in_critical())

;

}

// No cache readers in progress - garbage is now deletable

// Log our progress

if (PrintCaches) {

cache_collections++;

_objc_inform ("CACHES: COLLECTING %zu bytes (%zu regions, %zu allocations, %zu collections)", garbage_byte_size, cache_allocator_regions, cache_allocations, cache_collections);

}

// Dispose all refs now in the garbage

while (garbage_count--) {

_cache_free_block(garbage_refs[garbage_count]);

}

// Clear the garbage count and total size indicator

garbage_count = 0;

garbage_byte_size = 0;

if (PrintCaches) {

size_t i;

size_t total = 0;

size_t ideal_total = 0;

size_t malloc_total = 0;

size_t local_total = 0;

for (i = 0; i < sizeof(cache_counts) / sizeof(cache_counts[0]); i++) {

int count = cache_counts[i];

int slots = 1 << i;

size_t size = sizeof(struct objc_cache) + TABLE_SIZE(slots);

size_t ideal = size;

#if TARGET_OS_WIN32

size_t malloc = size;

#else

size_t malloc = malloc_good_size(size);

#endif

size_t local = size < CACHE_ALLOCATOR_MIN ? malloc : cache_allocator_size_for_mask(cache_allocator_mask_for_size(size));

if (!count) continue;

_objc_inform("CACHES: %4d slots: %4d caches, %6zu / %6zu / %6zu bytes ideal/malloc/local, %6zu / %6zu bytes wasted malloc/local", slots, count, ideal*count, malloc*count, local*count, malloc*count-ideal*count, local*count-ideal*count);

total += count;

ideal_total += ideal*count;

malloc_total += malloc*count;

local_total += local*count;

}

_objc_inform("CACHES: total: %4zu caches, %6zu / %6zu / %6zu bytes ideal/malloc/local, %6zu / %6zu bytes wasted malloc/local", total, ideal_total, malloc_total, local_total, malloc_total-ideal_total, local_total-ideal_total);

}

}

#if defined(CACHE_ALLOCATOR)

typedef struct cache_allocator_block {

uintptr_t size;

uintptr_t state;

struct cache_allocator_block *nextFree;

} cache_allocator_block;

typedef struct cache_allocator_region {

cache_allocator_block *start;

cache_allocator_block *end; // first non-block address

cache_allocator_block *freeList;

struct cache_allocator_region *next;

} cache_allocator_region;

static cache_allocator_region *cacheRegion = NULL;

static cache_allocator_region *cache_allocator_add_region(size_t size)

{

vm_address_t addr;

cache_allocator_block *b;

cache_allocator_region **rgnP;

cache_allocator_region *newRegion = (cache_allocator_region *)

calloc(1, sizeof(cache_allocator_region));

// Round size up to quantum boundary, and apply the minimum size.

size += CACHE_QUANTUM - (size % CACHE_QUANTUM);

if (size < CACHE_REGION_SIZE) size = CACHE_REGION_SIZE;

// Allocate the region

addr = (vm_address_t)calloc(size, 1);

newRegion->start = (cache_allocator_block *)addr;

newRegion->end = (cache_allocator_block *)(addr + size);

// Mark the first block: free and covers the entire region

b = newRegion->start;

b->size = size;

b->state = (uintptr_t)-1;

b->nextFree = NULL;

newRegion->freeList = b;