#define JEMALLOC_CTL_C_

#include "jemalloc/internal/jemalloc_internal.h"

static malloc_mutex_t ctl_mtx;

static bool ctl_initialized;

static uint64_t ctl_epoch;

static ctl_stats_t ctl_stats;

JEMALLOC_INLINE_C const ctl_named_node_t *

ctl_named_node(const ctl_node_t *node)

{

return ((node->named) ? (const ctl_named_node_t *)node : NULL);

}

JEMALLOC_INLINE_C const ctl_named_node_t *

ctl_named_children(const ctl_named_node_t *node, size_t index)

{

const ctl_named_node_t *children = ctl_named_node(node->children);

return (children ? &children[index] : NULL);

}

JEMALLOC_INLINE_C const ctl_indexed_node_t *

ctl_indexed_node(const ctl_node_t *node)

{

return (!node->named ? (const ctl_indexed_node_t *)node : NULL);

}

#define CTL_PROTO(n) \

static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \

void *oldp, size_t *oldlenp, void *newp, size_t newlen);

#define INDEX_PROTO(n) \

static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \

const size_t *mib, size_t miblen, size_t i);

static bool ctl_arena_init(ctl_arena_stats_t *astats);

static void ctl_arena_clear(ctl_arena_stats_t *astats);

static void ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats,

arena_t *arena);

static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,

ctl_arena_stats_t *astats);

static void ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i);

static bool ctl_grow(tsdn_t *tsdn);

static void ctl_refresh(tsdn_t *tsdn);

static bool ctl_init(tsdn_t *tsdn);

static int ctl_lookup(tsdn_t *tsdn, const char *name,

ctl_node_t const **nodesp, size_t *mibp, size_t *depthp);

CTL_PROTO(version)

CTL_PROTO(epoch)

static void arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all);

CTL_PROTO(arena_i_purge)

#define NAME(n) {true}, n

#define CHILD(t, c) \

sizeof(c##_node) / sizeof(ctl_##t##_node_t), \

(ctl_node_t *)c##_node, \

NULL

#define CTL(c) 0, NULL, c##_ctl

#define INDEX(i) {false}, i##_index

static const ctl_named_node_t thread_tcache_node[] = {

{NAME("enabled"), CTL(thread_tcache_enabled)},

{NAME("flush"), CTL(thread_tcache_flush)}

};

static const ctl_named_node_t thread_prof_node[] = {

{NAME("name"), CTL(thread_prof_name)},

{NAME("active"), CTL(thread_prof_active)}

};

static const ctl_named_node_t thread_node[] = {

{NAME("arena"), CTL(thread_arena)},

{NAME("allocated"), CTL(thread_allocated)},

{NAME("allocatedp"), CTL(thread_allocatedp)},

{NAME("deallocated"), CTL(thread_deallocated)},

{NAME("deallocatedp"), CTL(thread_deallocatedp)},

{NAME("tcache"), CHILD(named, thread_tcache)},

{NAME("prof"), CHILD(named, thread_prof)}

};

static const ctl_named_node_t config_node[] = {

{NAME("cache_oblivious"), CTL(config_cache_oblivious)},

{NAME("debug"), CTL(config_debug)},

{NAME("fill"), CTL(config_fill)},

{NAME("lazy_lock"), CTL(config_lazy_lock)},

{NAME("malloc_conf"), CTL(config_malloc_conf)},

{NAME("munmap"), CTL(config_munmap)},

{NAME("prof"), CTL(config_prof)},

{NAME("prof_libgcc"), CTL(config_prof_libgcc)},

{NAME("prof_libunwind"), CTL(config_prof_libunwind)},

{NAME("stats"), CTL(config_stats)},

{NAME("tcache"), CTL(config_tcache)},

{NAME("tls"), CTL(config_tls)},

{NAME("utrace"), CTL(config_utrace)},

{NAME("valgrind"), CTL(config_valgrind)},

{NAME("xmalloc"), CTL(config_xmalloc)}

};

static const ctl_named_node_t opt_node[] = {

{NAME("abort"), CTL(opt_abort)},

{NAME("dss"), CTL(opt_dss)},

{NAME("lg_chunk"), CTL(opt_lg_chunk)},

{NAME("narenas"), CTL(opt_narenas)},

{NAME("purge"), CTL(opt_purge)},

{NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)},

{NAME("decay_time"), CTL(opt_decay_time)},

{NAME("stats_print"), CTL(opt_stats_print)},

{NAME("junk"), CTL(opt_junk)},

{NAME("zero"), CTL(opt_zero)},

{NAME("quarantine"), CTL(opt_quarantine)},

{NAME("redzone"), CTL(opt_redzone)},

{NAME("utrace"), CTL(opt_utrace)},

{NAME("xmalloc"), CTL(opt_xmalloc)},

{NAME("tcache"), CTL(opt_tcache)},

{NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},

{NAME("prof"), CTL(opt_prof)},

{NAME("prof_prefix"), CTL(opt_prof_prefix)},

{NAME("prof_active"), CTL(opt_prof_active)},

{NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},

{NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},

{NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},

{NAME("prof_gdump"), CTL(opt_prof_gdump)},

{NAME("prof_final"), CTL(opt_prof_final)},

{NAME("prof_leak"), CTL(opt_prof_leak)},

{NAME("prof_accum"), CTL(opt_prof_accum)}

};

static const ctl_named_node_t tcache_node[] = {

{NAME("create"), CTL(tcache_create)},

{NAME("flush"), CTL(tcache_flush)},

{NAME("destroy"), CTL(tcache_destroy)}

};

static const ctl_named_node_t arena_i_node[] = {

{NAME("purge"), CTL(arena_i_purge)},

{NAME("decay"), CTL(arena_i_decay)},

{NAME("reset"), CTL(arena_i_reset)},

{NAME("dss"), CTL(arena_i_dss)},

{NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)},

{NAME("decay_time"), CTL(arena_i_decay_time)},

{NAME("chunk_hooks"), CTL(arena_i_chunk_hooks)}

};

static const ctl_named_node_t super_arena_i_node[] = {

{NAME(""), CHILD(named, arena_i)}

};

static const ctl_indexed_node_t arena_node[] = {

{INDEX(arena_i)}

};

static const ctl_named_node_t arenas_bin_i_node[] = {

{NAME("size"), CTL(arenas_bin_i_size)},

{NAME("nregs"), CTL(arenas_bin_i_nregs)},

{NAME("run_size"), CTL(arenas_bin_i_run_size)}

};

static const ctl_named_node_t super_arenas_bin_i_node[] = {

{NAME(""), CHILD(named, arenas_bin_i)}

};

static const ctl_indexed_node_t arenas_bin_node[] = {

{INDEX(arenas_bin_i)}

};

static const ctl_named_node_t arenas_lrun_i_node[] = {

{NAME("size"), CTL(arenas_lrun_i_size)}

};

static const ctl_named_node_t super_arenas_lrun_i_node[] = {

{NAME(""), CHILD(named, arenas_lrun_i)}

};

static const ctl_indexed_node_t arenas_lrun_node[] = {

{INDEX(arenas_lrun_i)}

};

static const ctl_named_node_t arenas_hchunk_i_node[] = {

{NAME("size"), CTL(arenas_hchunk_i_size)}

};

static const ctl_named_node_t super_arenas_hchunk_i_node[] = {

{NAME(""), CHILD(named, arenas_hchunk_i)}

};

static const ctl_indexed_node_t arenas_hchunk_node[] = {

{INDEX(arenas_hchunk_i)}

};

static const ctl_named_node_t arenas_node[] = {

{NAME("narenas"), CTL(arenas_narenas)},

{NAME("initialized"), CTL(arenas_initialized)},

{NAME("lg_dirty_mult"), CTL(arenas_lg_dirty_mult)},

{NAME("decay_time"), CTL(arenas_decay_time)},

{NAME("quantum"), CTL(arenas_quantum)},

{NAME("page"), CTL(arenas_page)},

{NAME("tcache_max"), CTL(arenas_tcache_max)},

{NAME("nbins"), CTL(arenas_nbins)},

{NAME("nhbins"), CTL(arenas_nhbins)},

{NAME("bin"), CHILD(indexed, arenas_bin)},

{NAME("nlruns"), CTL(arenas_nlruns)},

{NAME("lrun"), CHILD(indexed, arenas_lrun)},

{NAME("nhchunks"), CTL(arenas_nhchunks)},

{NAME("hchunk"), CHILD(indexed, arenas_hchunk)},

{NAME("extend"), CTL(arenas_extend)}

};

static const ctl_named_node_t prof_node[] = {

{NAME("thread_active_init"), CTL(prof_thread_active_init)},

{NAME("active"), CTL(prof_active)},

{NAME("dump"), CTL(prof_dump)},

{NAME("gdump"), CTL(prof_gdump)},

{NAME("reset"), CTL(prof_reset)},

{NAME("interval"), CTL(prof_interval)},

{NAME("lg_sample"), CTL(lg_prof_sample)}

};

static const ctl_named_node_t stats_arenas_i_metadata_node[] = {

{NAME("mapped"), CTL(stats_arenas_i_metadata_mapped)},

{NAME("allocated"), CTL(stats_arenas_i_metadata_allocated)}

};

static const ctl_named_node_t stats_arenas_i_small_node[] = {

{NAME("allocated"), CTL(stats_arenas_i_small_allocated)},

{NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}

};

static const ctl_named_node_t stats_arenas_i_large_node[] = {

{NAME("allocated"), CTL(stats_arenas_i_large_allocated)},

{NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}

};

static const ctl_named_node_t stats_arenas_i_huge_node[] = {

{NAME("allocated"), CTL(stats_arenas_i_huge_allocated)},

{NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)}

};

static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {

{NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)},

{NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)},

{NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)},

{NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)},

{NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)},

{NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)},

{NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)}

};

static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {

{NAME(""), CHILD(named, stats_arenas_i_bins_j)}

};

static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {

{INDEX(stats_arenas_i_bins_j)}

};

static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = {

{NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)},

{NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)}

};

static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = {

{NAME(""), CHILD(named, stats_arenas_i_lruns_j)}

};

static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = {

{INDEX(stats_arenas_i_lruns_j)}

};

static const ctl_named_node_t stats_arenas_i_hchunks_j_node[] = {

{NAME("nmalloc"), CTL(stats_arenas_i_hchunks_j_nmalloc)},

{NAME("ndalloc"), CTL(stats_arenas_i_hchunks_j_ndalloc)},

{NAME("nrequests"), CTL(stats_arenas_i_hchunks_j_nrequests)},

{NAME("curhchunks"), CTL(stats_arenas_i_hchunks_j_curhchunks)}

};

static const ctl_named_node_t super_stats_arenas_i_hchunks_j_node[] = {

{NAME(""), CHILD(named, stats_arenas_i_hchunks_j)}

};

static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = {

{INDEX(stats_arenas_i_hchunks_j)}

};

static const ctl_named_node_t stats_arenas_i_node[] = {

{NAME("nthreads"), CTL(stats_arenas_i_nthreads)},

{NAME("dss"), CTL(stats_arenas_i_dss)},

{NAME("lg_dirty_mult"), CTL(stats_arenas_i_lg_dirty_mult)},

{NAME("decay_time"), CTL(stats_arenas_i_decay_time)},

{NAME("pactive"), CTL(stats_arenas_i_pactive)},

{NAME("pdirty"), CTL(stats_arenas_i_pdirty)},

{NAME("mapped"), CTL(stats_arenas_i_mapped)},

{NAME("retained"), CTL(stats_arenas_i_retained)},

{NAME("npurge"), CTL(stats_arenas_i_npurge)},

{NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)},

{NAME("purged"), CTL(stats_arenas_i_purged)},

{NAME("metadata"), CHILD(named, stats_arenas_i_metadata)},

{NAME("small"), CHILD(named, stats_arenas_i_small)},

{NAME("large"), CHILD(named, stats_arenas_i_large)},

{NAME("huge"), CHILD(named, stats_arenas_i_huge)},

{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},

{NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)},

{NAME("hchunks"), CHILD(indexed, stats_arenas_i_hchunks)}

};

static const ctl_named_node_t super_stats_arenas_i_node[] = {

{NAME(""), CHILD(named, stats_arenas_i)}

};

static const ctl_indexed_node_t stats_arenas_node[] = {

{INDEX(stats_arenas_i)}

};

static const ctl_named_node_t stats_node[] = {

{NAME("cactive"), CTL(stats_cactive)},

{NAME("allocated"), CTL(stats_allocated)},

{NAME("active"), CTL(stats_active)},

{NAME("metadata"), CTL(stats_metadata)},

{NAME("resident"), CTL(stats_resident)},

{NAME("mapped"), CTL(stats_mapped)},

{NAME("retained"), CTL(stats_retained)},

{NAME("arenas"), CHILD(indexed, stats_arenas)}

};

static const ctl_named_node_t root_node[] = {

{NAME("version"), CTL(version)},

{NAME("epoch"), CTL(epoch)},

{NAME("thread"), CHILD(named, thread)},

{NAME("config"), CHILD(named, config)},

{NAME("opt"), CHILD(named, opt)},

{NAME("tcache"), CHILD(named, tcache)},

{NAME("arena"), CHILD(indexed, arena)},

{NAME("arenas"), CHILD(named, arenas)},

{NAME("prof"), CHILD(named, prof)},

{NAME("stats"), CHILD(named, stats)}

};

static const ctl_named_node_t super_root_node[] = {

{NAME(""), CHILD(named, root)}

};

#undef NAME

#undef CHILD

#undef CTL

#undef INDEX

static bool

ctl_arena_init(ctl_arena_stats_t *astats)

{

if (astats->lstats == NULL) {

astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses *

sizeof(malloc_large_stats_t));

if (astats->lstats == NULL)

return (true);

}

if (astats->hstats == NULL) {

astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses *

sizeof(malloc_huge_stats_t));

if (astats->hstats == NULL)

return (true);

}

return (false);

}

static void

ctl_arena_clear(ctl_arena_stats_t *astats)

{

astats->nthreads = 0;

astats->dss = dss_prec_names[dss_prec_limit];

astats->lg_dirty_mult = -1;

astats->decay_time = -1;

astats->pactive = 0;

astats->pdirty = 0;

if (config_stats) {

memset(&astats->astats, 0, sizeof(arena_stats_t));

astats->allocated_small = 0;

astats->nmalloc_small = 0;

astats->ndalloc_small = 0;

astats->nrequests_small = 0;

memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t));

memset(astats->lstats, 0, nlclasses *

sizeof(malloc_large_stats_t));

memset(astats->hstats, 0, nhclasses *

sizeof(malloc_huge_stats_t));

}

}

static void

ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_stats_t *cstats, arena_t *arena)

{

unsigned i;

if (config_stats) {

arena_stats_merge(tsdn, arena, &cstats->nthreads, &cstats->dss,

&cstats->lg_dirty_mult, &cstats->decay_time,

&cstats->pactive, &cstats->pdirty, &cstats->astats,

cstats->bstats, cstats->lstats, cstats->hstats);

for (i = 0; i < NBINS; i++) {

cstats->allocated_small += cstats->bstats[i].curregs *

index2size(i);

cstats->nmalloc_small += cstats->bstats[i].nmalloc;

cstats->ndalloc_small += cstats->bstats[i].ndalloc;

cstats->nrequests_small += cstats->bstats[i].nrequests;

}

} else {

arena_basic_stats_merge(tsdn, arena, &cstats->nthreads,

&cstats->dss, &cstats->lg_dirty_mult, &cstats->decay_time,

&cstats->pactive, &cstats->pdirty);

}

}

static void

ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)

{

unsigned i;

sstats->nthreads += astats->nthreads;

sstats->pactive += astats->pactive;

sstats->pdirty += astats->pdirty;

if (config_stats) {

sstats->astats.mapped += astats->astats.mapped;

sstats->astats.retained += astats->astats.retained;

sstats->astats.npurge += astats->astats.npurge;

sstats->astats.nmadvise += astats->astats.nmadvise;

sstats->astats.purged += astats->astats.purged;

sstats->astats.metadata_mapped +=

astats->astats.metadata_mapped;

sstats->astats.metadata_allocated +=

astats->astats.metadata_allocated;

sstats->allocated_small += astats->allocated_small;

sstats->nmalloc_small += astats->nmalloc_small;

sstats->ndalloc_small += astats->ndalloc_small;

sstats->nrequests_small += astats->nrequests_small;

sstats->astats.allocated_large +=

astats->astats.allocated_large;

sstats->astats.nmalloc_large += astats->astats.nmalloc_large;

sstats->astats.ndalloc_large += astats->astats.ndalloc_large;

sstats->astats.nrequests_large +=

astats->astats.nrequests_large;

sstats->astats.allocated_huge += astats->astats.allocated_huge;

sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge;

sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge;

for (i = 0; i < NBINS; i++) {

sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;

sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;

sstats->bstats[i].nrequests +=

astats->bstats[i].nrequests;

sstats->bstats[i].curregs += astats->bstats[i].curregs;

if (config_tcache) {

sstats->bstats[i].nfills +=

astats->bstats[i].nfills;

sstats->bstats[i].nflushes +=

astats->bstats[i].nflushes;

}

sstats->bstats[i].nruns += astats->bstats[i].nruns;

sstats->bstats[i].reruns += astats->bstats[i].reruns;

sstats->bstats[i].curruns += astats->bstats[i].curruns;

}

for (i = 0; i < nlclasses; i++) {

sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;

sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;

sstats->lstats[i].nrequests +=

astats->lstats[i].nrequests;

sstats->lstats[i].curruns += astats->lstats[i].curruns;

}

for (i = 0; i < nhclasses; i++) {

sstats->hstats[i].nmalloc += astats->hstats[i].nmalloc;

sstats->hstats[i].ndalloc += astats->hstats[i].ndalloc;

sstats->hstats[i].curhchunks +=

astats->hstats[i].curhchunks;

}

}

}

static void

ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)

{

ctl_arena_stats_t *astats = &ctl_stats.arenas[i];

ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas];

ctl_arena_clear(astats);

ctl_arena_stats_amerge(tsdn, astats, arena);

/* Merge into sum stats as well. */

ctl_arena_stats_smerge(sstats, astats);

}

static bool

ctl_grow(tsdn_t *tsdn)

{

ctl_arena_stats_t *astats;

/* Initialize new arena. */

if (arena_init(tsdn, ctl_stats.narenas) == NULL)

return (true);

/* Allocate extended arena stats. */

astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) *

sizeof(ctl_arena_stats_t));

if (astats == NULL)

return (true);

/* Initialize the new astats element. */

memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *

sizeof(ctl_arena_stats_t));

memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));

if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) {

a0dalloc(astats);

return (true);

}

/* Swap merged stats to their new location. */

{

ctl_arena_stats_t tstats;

memcpy(&tstats, &astats[ctl_stats.narenas],

sizeof(ctl_arena_stats_t));

memcpy(&astats[ctl_stats.narenas],

&astats[ctl_stats.narenas + 1], sizeof(ctl_arena_stats_t));

memcpy(&astats[ctl_stats.narenas + 1], &tstats,

sizeof(ctl_arena_stats_t));

}

a0dalloc(ctl_stats.arenas);

ctl_stats.arenas = astats;

ctl_stats.narenas++;

return (false);

}

static void

ctl_refresh(tsdn_t *tsdn)

{

unsigned i;

VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);

/*

ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]);

for (i = 0; i < ctl_stats.narenas; i++)

tarenas[i] = arena_get(tsdn, i, false);

for (i = 0; i < ctl_stats.narenas; i++) {

bool initialized = (tarenas[i] != NULL);

ctl_stats.arenas[i].initialized = initialized;

if (initialized)

ctl_arena_refresh(tsdn, tarenas[i], i);

}

if (config_stats) {

size_t base_allocated, base_resident, base_mapped;

base_stats_get(tsdn, &base_allocated, &base_resident,

&base_mapped);

ctl_stats.allocated =

ctl_stats.arenas[ctl_stats.narenas].allocated_small +

ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large +

ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge;

ctl_stats.active =

(ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE);

ctl_stats.metadata = base_allocated +

ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped +

ctl_stats.arenas[ctl_stats.narenas].astats

.metadata_allocated;

ctl_stats.resident = base_resident +

ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped +

((ctl_stats.arenas[ctl_stats.narenas].pactive +

ctl_stats.arenas[ctl_stats.narenas].pdirty) << LG_PAGE);

ctl_stats.mapped = base_mapped +

ctl_stats.arenas[ctl_stats.narenas].astats.mapped;

ctl_stats.retained =

ctl_stats.arenas[ctl_stats.narenas].astats.retained;

}

ctl_epoch++;

}

static bool

ctl_init(tsdn_t *tsdn)

{

bool ret;

malloc_mutex_lock(tsdn, &ctl_mtx);

if (!ctl_initialized) {

/*

ctl_stats.narenas = narenas_total_get();

ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc(

(ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t));

if (ctl_stats.arenas == NULL) {

ret = true;

goto label_return;

}

memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) *

sizeof(ctl_arena_stats_t));

/*

if (config_stats) {

unsigned i;

for (i = 0; i <= ctl_stats.narenas; i++) {

if (ctl_arena_init(&ctl_stats.arenas[i])) {

unsigned j;

for (j = 0; j < i; j++) {

a0dalloc(

ctl_stats.arenas[j].lstats);

a0dalloc(

ctl_stats.arenas[j].hstats);

}

a0dalloc(ctl_stats.arenas);

ctl_stats.arenas = NULL;

ret = true;

goto label_return;

}

}

}

ctl_stats.arenas[ctl_stats.narenas].initialized = true;

ctl_epoch = 0;

ctl_refresh(tsdn);

ctl_initialized = true;

}

ret = false;

label_return:

malloc_mutex_unlock(tsdn, &ctl_mtx);

return (ret);

}

static int

ctl_lookup(tsdn_t *tsdn, const char *name, ctl_node_t const **nodesp,

size_t *mibp, size_t *depthp)

{

int ret;

const char *elm, *tdot, *dot;

size_t elen, i, j;

const ctl_named_node_t *node;

elm = name;

/* Equivalent to strchrnul(). */

dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0');

elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);

if (elen == 0) {

ret = ENOENT;

goto label_return;

}

node = super_root_node;

for (i = 0; i < *depthp; i++) {

assert(node);

assert(node->nchildren > 0);

if (ctl_named_node(node->children) != NULL) {

const ctl_named_node_t *pnode = node;

/* Children are named. */

for (j = 0; j < node->nchildren; j++) {

const ctl_named_node_t *child =

ctl_named_children(node, j);

if (strlen(child->name) == elen &&

strncmp(elm, child->name, elen) == 0) {

node = child;

if (nodesp != NULL)

nodesp[i] =

(const ctl_node_t *)node;

mibp[i] = j;

break;

}

}

if (node == pnode) {

ret = ENOENT;

goto label_return;

}

} else {

uintmax_t index;

const ctl_indexed_node_t *inode;

/* Children are indexed. */

index = malloc_strtoumax(elm, NULL, 10);

if (index == UINTMAX_MAX || index > SIZE_T_MAX) {

ret = ENOENT;

goto label_return;

}

inode = ctl_indexed_node(node->children);

node = inode->index(tsdn, mibp, *depthp, (size_t)index);

if (node == NULL) {

ret = ENOENT;

goto label_return;

}

if (nodesp != NULL)

nodesp[i] = (const ctl_node_t *)node;

mibp[i] = (size_t)index;

}

if (node->ctl != NULL) {

/* Terminal node. */

if (*dot != '\0') {

/*

ret = ENOENT;

goto label_return;

}

/* Complete lookup successful. */

*depthp = i + 1;

break;

}

/* Update elm. */

if (*dot == '\0') {

/* No more elements. */

ret = ENOENT;

goto label_return;

}

elm = &dot[1];

dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot :

strchr(elm, '\0');

elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);

}

ret = 0;

label_return:

return (ret);

}

ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,

void *newp, size_t newlen)

{

int ret;

size_t depth;

ctl_node_t const *nodes[CTL_MAX_DEPTH];

size_t mib[CTL_MAX_DEPTH];

const ctl_named_node_t *node;

if (!ctl_initialized && ctl_init(tsd_tsdn(tsd))) {

ret = EAGAIN;

goto label_return;

}

depth = CTL_MAX_DEPTH;

ret = ctl_lookup(tsd_tsdn(tsd), name, nodes, mib, &depth);

if (ret != 0)

goto label_return;

node = ctl_named_node(nodes[depth-1]);

if (node != NULL && node->ctl)

ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);

else {

/* The name refers to a partial path through the ctl tree. */

ret = ENOENT;

}

label_return:

return(ret);

}

ctl_nametomib(tsdn_t *tsdn, const char *name, size_t *mibp, size_t *miblenp)

{

int ret;

if (!ctl_initialized && ctl_init(tsdn)) {

ret = EAGAIN;

goto label_return;

}

ret = ctl_lookup(tsdn, name, NULL, mibp, miblenp);

label_return:

return(ret);

}

ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,

size_t *oldlenp, void *newp, size_t newlen)

{

int ret;

const ctl_named_node_t *node;

size_t i;

if (!ctl_initialized && ctl_init(tsd_tsdn(tsd))) {

ret = EAGAIN;

goto label_return;

}

/* Iterate down the tree. */

node = super_root_node;

for (i = 0; i < miblen; i++) {

assert(node);

assert(node->nchildren > 0);

if (ctl_named_node(node->children) != NULL) {

/* Children are named. */

if (node->nchildren <= (unsigned)mib[i]) {

ret = ENOENT;

goto label_return;

}

node = ctl_named_children(node, mib[i]);

} else {

const ctl_indexed_node_t *inode;

/* Indexed element. */

inode = ctl_indexed_node(node->children);

node = inode->index(tsd_tsdn(tsd), mib, miblen, mib[i]);

if (node == NULL) {

ret = ENOENT;

goto label_return;

}

}

}

/* Call the ctl function. */

if (node && node->ctl)