jemalloc-jemalloc/include/jemalloc/internal/tcache.h

#ifndef JEMALLOC_INTERNAL_TCACHE_H
#define JEMALLOC_INTERNAL_TCACHE_H

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/cache_bin.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event_registry.h"
#include "jemalloc/internal/ticker.h"

/* Forward decls; only used as pointer types below. */
typedef struct arena_s arena_t;
typedef struct base_s  base_t;

/******************************************************************************/
/* TYPES */
/******************************************************************************/

typedef struct tcache_slow_s tcache_slow_t;
typedef struct tcache_s      tcache_t;
typedef struct tcaches_s     tcaches_t;

/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
#define TCACHE_ZERO_INITIALIZER                                                \
	{ 0 }
#define TCACHE_SLOW_ZERO_INITIALIZER                                           \
	{                                                                      \
		{ 0 }                                                          \
	}

/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
#define TCACHE_ENABLED_ZERO_INITIALIZER false

/* Used for explicit tcache only. Means flushed but not destroyed. */
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)

#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
#define TCACHE_NBINS_MAX                                                       \
	(SC_NBINS                                                              \
	    + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS)    \
	    + 1)
#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21)       /* 2M */
#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
#define TCACHE_GC_LARGE_NBINS_MAX 1

/******************************************************************************/
/* STRUCTS */
/******************************************************************************/

/*
 * The tcache state is split into the slow and hot path data.  Each has a
 * pointer to the other, and the data always comes in pairs.  The layout of each
 * of them varies in practice; tcache_slow lives in the TSD for the automatic
 * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
 * a pointer to tcache_slow lets us treat these cases uniformly, rather than
 * splitting up the tcache [de]allocation code into those paths called with the
 * TSD tcache and those called with a manual tcache.
 */

struct tcache_slow_s {
	/*
	 * The descriptor lets the arena find our cache bins without seeing the
	 * tcache definition.  This enables arenas to aggregate stats across
	 * tcaches without having a tcache dependency.
	 */
	cache_bin_array_descriptor_t cache_bin_array_descriptor;

	/* The arena this tcache is associated with. */
	arena_t *arena;
	/* The number of bins activated in the tcache. */
	unsigned tcache_nbins;
	/* Last time GC has been performed.  */
	nstime_t last_gc_time;
	/* Next bin to GC. */
	szind_t next_gc_bin;
	szind_t next_gc_bin_small;
	szind_t next_gc_bin_large;
	/* For small bins, help determine how many items to fill at a time. */
	cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
	/* For small bins, whether has been refilled since last GC. */
	bool bin_refilled[SC_NBINS];
	/*
	 * For small bins, the number of items we can pretend to flush before
	 * actually flushing.
	 */
	uint8_t bin_flush_delay_items[SC_NBINS];
	/*
	 * The start of the allocation containing the dynamic allocation for
	 * either the cache bins alone, or the cache bin memory as well as this
	 * tcache_slow_t and its associated tcache_t.
	 */
	void *dyn_alloc;

	/* The associated bins. */
	tcache_t *tcache;
};

struct tcache_s {
	tcache_slow_t *tcache_slow;
	cache_bin_t    bins[TCACHE_NBINS_MAX];
};

/* Linkage for list of available (previously used) explicit tcache IDs. */
struct tcaches_s {
	union {
		tcache_t  *tcache;
		tcaches_t *next;
	};
};

/******************************************************************************/
/* EXTERNS */
/******************************************************************************/

extern bool     opt_tcache;
extern size_t   opt_tcache_max;
extern ssize_t  opt_lg_tcache_nslots_mul;
extern unsigned opt_tcache_nslots_small_min;
extern unsigned opt_tcache_nslots_small_max;
extern unsigned opt_tcache_nslots_large;
extern ssize_t  opt_lg_tcache_shift;
extern size_t   opt_tcache_gc_incr_bytes;
extern size_t   opt_tcache_gc_delay_bytes;
extern unsigned opt_lg_tcache_flush_small_div;
extern unsigned opt_lg_tcache_flush_large_div;

/*
 * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more
 * large-object bins.  This is only used during threads initialization and
 * changing it will not reflect on initialized threads as expected.  Thus,
 * it should not be changed on the fly.  To change the number of tcache bins
 * in use, refer to tcache_nbins of each tcache.
 */
extern unsigned global_do_not_change_tcache_nbins;

/*
 * Maximum cached size class.  Same as above, this is only used during threads
 * initialization and should not be changed.  To change the maximum cached size
 * class, refer to tcache_max of each tcache.
 */
extern size_t global_do_not_change_tcache_maxclass;

/*
 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
 * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
 * completely disjoint from this data structure.  tcaches starts off as a sparse
 * array, so it has no physical memory footprint until individual pages are
 * touched.  This allows the entire array to be allocated the first time an
 * explicit tcache is created without a disproportionate impact on memory usage.
 */
extern tcaches_t *tcaches;

size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
void  *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
     cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);

void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
    cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_default_init(
    const char *bin_settings_segment_cur, size_t len_left);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(
    tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
void tcache_arena_reassociate(
    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
tcache_t *tcache_create_explicit(tsd_t *tsd);
bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
void      tcache_cleanup(tsd_t *tsd);
bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
void      tcaches_flush(tsd_t *tsd, unsigned ind);
void      tcaches_destroy(tsd_t *tsd, unsigned ind);
bool      tcache_boot(tsdn_t *tsdn, base_t *base);
void      tcache_arena_associate(
         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
         tsdn_t *tsdn, arena_t *arena);
void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn);
void tcache_flush(tsd_t *tsd);
bool tsd_tcache_enabled_data_init(tsd_t *tsd);
void tcache_enabled_set(tsd_t *tsd, bool enabled);

extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
    size_t alignment);

void tcache_assert_initialized(tcache_t *tcache);

extern te_base_cb_t tcache_gc_te_handler;

#endif /* JEMALLOC_INTERNAL_TCACHE_H */