Commit e1e4b751 authored by Michael R. Crusoe's avatar Michael R. Crusoe 🏳️‍🌈
Browse files

New upstream version 3.13.0+dfsg2

parent 67452d60

ext/include/jemalloc/internal/arena.h

0 → 100644
+975 −0

File added.

Preview size limit exceeded, changes collapsed.

ext/include/jemalloc/internal/atomic.h

0 → 100644
+304 −0
Original line number Diff line number Diff line 
/******************************************************************************/

#ifdef JEMALLOC_H_TYPES



#endif /* JEMALLOC_H_TYPES */

/******************************************************************************/

#ifdef JEMALLOC_H_STRUCTS



#endif /* JEMALLOC_H_STRUCTS */

/******************************************************************************/

#ifdef JEMALLOC_H_EXTERNS



#define	atomic_read_uint64(p)	atomic_add_uint64(p, 0)

#define	atomic_read_uint32(p)	atomic_add_uint32(p, 0)

#define	atomic_read_z(p)	atomic_add_z(p, 0)

#define	atomic_read_u(p)	atomic_add_u(p, 0)



#endif /* JEMALLOC_H_EXTERNS */

/******************************************************************************/

#ifdef JEMALLOC_H_INLINES



#ifndef JEMALLOC_ENABLE_INLINE

uint64_t	atomic_add_uint64(uint64_t *p, uint64_t x);

uint64_t	atomic_sub_uint64(uint64_t *p, uint64_t x);

uint32_t	atomic_add_uint32(uint32_t *p, uint32_t x);

uint32_t	atomic_sub_uint32(uint32_t *p, uint32_t x);

size_t	atomic_add_z(size_t *p, size_t x);

size_t	atomic_sub_z(size_t *p, size_t x);

unsigned	atomic_add_u(unsigned *p, unsigned x);

unsigned	atomic_sub_u(unsigned *p, unsigned x);

#endif



#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))

/******************************************************************************/

/* 64-bit operations. */

#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)

#  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	return (__sync_add_and_fetch(p, x));

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	return (__sync_sub_and_fetch(p, x));

}

#elif (defined(_MSC_VER))

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	return (InterlockedExchangeAdd64(p, x));

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	return (InterlockedExchangeAdd64(p, -((int64_t)x)));

}

#elif (defined(JEMALLOC_OSATOMIC))

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	return (OSAtomicAdd64((int64_t)x, (int64_t *)p));

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));

}

#  elif (defined(__amd64__) || defined(__x86_64__))

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	asm volatile (

	    "lock; xaddq %0, %1;"

	    : "+r" (x), "=m" (*p) /* Outputs. */

	    : "m" (*p) /* Inputs. */

	    );



	return (x);

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	x = (uint64_t)(-(int64_t)x);

	asm volatile (

	    "lock; xaddq %0, %1;"

	    : "+r" (x), "=m" (*p) /* Outputs. */

	    : "m" (*p) /* Inputs. */

	    );



	return (x);

}

#  elif (defined(JEMALLOC_ATOMIC9))

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	/*

	 * atomic_fetchadd_64() doesn't exist, but we only ever use this

	 * function on LP64 systems, so atomic_fetchadd_long() will do.

	 */

	assert(sizeof(uint64_t) == sizeof(unsigned long));



	return (atomic_fetchadd_long(p, (unsigned long)x) + x);

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	assert(sizeof(uint64_t) == sizeof(unsigned long));



	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);

}

#  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))

JEMALLOC_INLINE uint64_t

atomic_add_uint64(uint64_t *p, uint64_t x)

{



	return (__sync_add_and_fetch(p, x));

}



JEMALLOC_INLINE uint64_t

atomic_sub_uint64(uint64_t *p, uint64_t x)

{



	return (__sync_sub_and_fetch(p, x));

}

#  else

#    error "Missing implementation for 64-bit atomic operations"

#  endif

#endif



/******************************************************************************/

/* 32-bit operations. */

#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	return (__sync_add_and_fetch(p, x));

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	return (__sync_sub_and_fetch(p, x));

}

#elif (defined(_MSC_VER))

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	return (InterlockedExchangeAdd(p, x));

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	return (InterlockedExchangeAdd(p, -((int32_t)x)));

}

#elif (defined(JEMALLOC_OSATOMIC))

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	return (OSAtomicAdd32((int32_t)x, (int32_t *)p));

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));

}

#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	asm volatile (

	    "lock; xaddl %0, %1;"

	    : "+r" (x), "=m" (*p) /* Outputs. */

	    : "m" (*p) /* Inputs. */

	    );



	return (x);

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	x = (uint32_t)(-(int32_t)x);

	asm volatile (

	    "lock; xaddl %0, %1;"

	    : "+r" (x), "=m" (*p) /* Outputs. */

	    : "m" (*p) /* Inputs. */

	    );



	return (x);

}

#elif (defined(JEMALLOC_ATOMIC9))

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	return (atomic_fetchadd_32(p, x) + x);

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);

}

#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))

JEMALLOC_INLINE uint32_t

atomic_add_uint32(uint32_t *p, uint32_t x)

{



	return (__sync_add_and_fetch(p, x));

}



JEMALLOC_INLINE uint32_t

atomic_sub_uint32(uint32_t *p, uint32_t x)

{



	return (__sync_sub_and_fetch(p, x));

}

#else

#  error "Missing implementation for 32-bit atomic operations"

#endif



/******************************************************************************/

/* size_t operations. */

JEMALLOC_INLINE size_t

atomic_add_z(size_t *p, size_t x)

{



#if (LG_SIZEOF_PTR == 3)

	return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));

#elif (LG_SIZEOF_PTR == 2)

	return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));

#endif

}



JEMALLOC_INLINE size_t

atomic_sub_z(size_t *p, size_t x)

{



#if (LG_SIZEOF_PTR == 3)

	return ((size_t)atomic_add_uint64((uint64_t *)p,

	    (uint64_t)-((int64_t)x)));

#elif (LG_SIZEOF_PTR == 2)

	return ((size_t)atomic_add_uint32((uint32_t *)p,

	    (uint32_t)-((int32_t)x)));

#endif

}



/******************************************************************************/

/* unsigned operations. */

JEMALLOC_INLINE unsigned

atomic_add_u(unsigned *p, unsigned x)

{



#if (LG_SIZEOF_INT == 3)

	return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));

#elif (LG_SIZEOF_INT == 2)

	return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));

#endif

}



JEMALLOC_INLINE unsigned

atomic_sub_u(unsigned *p, unsigned x)

{



#if (LG_SIZEOF_INT == 3)

	return ((unsigned)atomic_add_uint64((uint64_t *)p,

	    (uint64_t)-((int64_t)x)));

#elif (LG_SIZEOF_INT == 2)

	return ((unsigned)atomic_add_uint32((uint32_t *)p,

	    (uint32_t)-((int32_t)x)));

#endif

}

/******************************************************************************/

#endif



#endif /* JEMALLOC_H_INLINES */

/******************************************************************************/

ext/include/jemalloc/internal/base.h

0 → 100644
+26 −0
Original line number Diff line number Diff line 
/******************************************************************************/

#ifdef JEMALLOC_H_TYPES



#endif /* JEMALLOC_H_TYPES */

/******************************************************************************/

#ifdef JEMALLOC_H_STRUCTS



#endif /* JEMALLOC_H_STRUCTS */

/******************************************************************************/

#ifdef JEMALLOC_H_EXTERNS



void	*base_alloc(size_t size);

void	*base_calloc(size_t number, size_t size);

extent_node_t *base_node_alloc(void);

void	base_node_dealloc(extent_node_t *node);

bool	base_boot(void);

void	base_prefork(void);

void	base_postfork_parent(void);

void	base_postfork_child(void);



#endif /* JEMALLOC_H_EXTERNS */

/******************************************************************************/

#ifdef JEMALLOC_H_INLINES



#endif /* JEMALLOC_H_INLINES */

/******************************************************************************/

ext/include/jemalloc/internal/bitmap.h

0 → 100644
+184 −0
Original line number Diff line number Diff line 
/******************************************************************************/

#ifdef JEMALLOC_H_TYPES



/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */

#define	LG_BITMAP_MAXBITS	LG_RUN_MAXREGS



typedef struct bitmap_level_s bitmap_level_t;

typedef struct bitmap_info_s bitmap_info_t;

typedef unsigned long bitmap_t;

#define	LG_SIZEOF_BITMAP	LG_SIZEOF_LONG



/* Number of bits per group. */

#define	LG_BITMAP_GROUP_NBITS		(LG_SIZEOF_BITMAP + 3)

#define	BITMAP_GROUP_NBITS		(ZU(1) << LG_BITMAP_GROUP_NBITS)

#define	BITMAP_GROUP_NBITS_MASK		(BITMAP_GROUP_NBITS-1)



/* Maximum number of levels possible. */

#define	BITMAP_MAX_LEVELS						\

    (LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP)				\

    + !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)



#endif /* JEMALLOC_H_TYPES */

/******************************************************************************/

#ifdef JEMALLOC_H_STRUCTS



struct bitmap_level_s {

	/* Offset of this level's groups within the array of groups. */

	size_t group_offset;

};



struct bitmap_info_s {

	/* Logical number of bits in bitmap (stored at bottom level). */

	size_t nbits;



	/* Number of levels necessary for nbits. */

	unsigned nlevels;



	/*

	 * Only the first (nlevels+1) elements are used, and levels are ordered

	 * bottom to top (e.g. the bottom level is stored in levels[0]).

	 */

	bitmap_level_t levels[BITMAP_MAX_LEVELS+1];

};



#endif /* JEMALLOC_H_STRUCTS */

/******************************************************************************/

#ifdef JEMALLOC_H_EXTERNS



void	bitmap_info_init(bitmap_info_t *binfo, size_t nbits);

size_t	bitmap_info_ngroups(const bitmap_info_t *binfo);

size_t	bitmap_size(size_t nbits);

void	bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo);



#endif /* JEMALLOC_H_EXTERNS */

/******************************************************************************/

#ifdef JEMALLOC_H_INLINES



#ifndef JEMALLOC_ENABLE_INLINE

bool	bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo);

bool	bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);

void	bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);

size_t	bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo);

void	bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);

#endif



#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_))

JEMALLOC_INLINE bool

bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo)

{

	unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1;

	bitmap_t rg = bitmap[rgoff];

	/* The bitmap is full iff the root group is 0. */

	return (rg == 0);

}



JEMALLOC_INLINE bool

bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)

{

	size_t goff;

	bitmap_t g;



	assert(bit < binfo->nbits);

	goff = bit >> LG_BITMAP_GROUP_NBITS;

	g = bitmap[goff];

	return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))));

}



JEMALLOC_INLINE void

bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)

{

	size_t goff;

	bitmap_t *gp;

	bitmap_t g;



	assert(bit < binfo->nbits);

	assert(bitmap_get(bitmap, binfo, bit) == false);

	goff = bit >> LG_BITMAP_GROUP_NBITS;

	gp = &bitmap[goff];

	g = *gp;

	assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));

	g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);

	*gp = g;

	assert(bitmap_get(bitmap, binfo, bit));

	/* Propagate group state transitions up the tree. */

	if (g == 0) {

		unsigned i;

		for (i = 1; i < binfo->nlevels; i++) {

			bit = goff;

			goff = bit >> LG_BITMAP_GROUP_NBITS;

			gp = &bitmap[binfo->levels[i].group_offset + goff];

			g = *gp;

			assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));

			g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);

			*gp = g;

			if (g != 0)

				break;

		}

	}

}



/* sfu: set first unset. */

JEMALLOC_INLINE size_t

bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo)

{

	size_t bit;

	bitmap_t g;

	unsigned i;



	assert(bitmap_full(bitmap, binfo) == false);



	i = binfo->nlevels - 1;

	g = bitmap[binfo->levels[i].group_offset];

	bit = ffsl(g) - 1;

	while (i > 0) {

		i--;

		g = bitmap[binfo->levels[i].group_offset + bit];

		bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1);

	}



	bitmap_set(bitmap, binfo, bit);

	return (bit);

}



JEMALLOC_INLINE void

bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)

{

	size_t goff;

	bitmap_t *gp;

	bitmap_t g;

	bool propagate;



	assert(bit < binfo->nbits);

	assert(bitmap_get(bitmap, binfo, bit));

	goff = bit >> LG_BITMAP_GROUP_NBITS;

	gp = &bitmap[goff];

	g = *gp;

	propagate = (g == 0);

	assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);

	g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);

	*gp = g;

	assert(bitmap_get(bitmap, binfo, bit) == false);

	/* Propagate group state transitions up the tree. */

	if (propagate) {

		unsigned i;

		for (i = 1; i < binfo->nlevels; i++) {

			bit = goff;

			goff = bit >> LG_BITMAP_GROUP_NBITS;

			gp = &bitmap[binfo->levels[i].group_offset + goff];

			g = *gp;

			propagate = (g == 0);

			assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))

			    == 0);

			g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);

			*gp = g;

			if (propagate == false)

				break;

		}

	}

}



#endif



#endif /* JEMALLOC_H_INLINES */

/******************************************************************************/

ext/include/jemalloc/internal/chunk.h

0 → 100644
+63 −0
Original line number Diff line number Diff line 
/******************************************************************************/

#ifdef JEMALLOC_H_TYPES



/*

 * Size and alignment of memory chunks that are allocated by the OS's virtual

 * memory system.

 */

#define	LG_CHUNK_DEFAULT	22



/* Return the chunk address for allocation address a. */

#define	CHUNK_ADDR2BASE(a)						\

	((void *)((uintptr_t)(a) & ~chunksize_mask))



/* Return the chunk offset of address a. */

#define	CHUNK_ADDR2OFFSET(a)						\

	((size_t)((uintptr_t)(a) & chunksize_mask))



/* Return the smallest chunk multiple that is >= s. */

#define	CHUNK_CEILING(s)						\

	(((s) + chunksize_mask) & ~chunksize_mask)



#endif /* JEMALLOC_H_TYPES */

/******************************************************************************/

#ifdef JEMALLOC_H_STRUCTS



#endif /* JEMALLOC_H_STRUCTS */

/******************************************************************************/

#ifdef JEMALLOC_H_EXTERNS



extern size_t		opt_lg_chunk;

extern const char	*opt_dss;



/* Protects stats_chunks; currently not used for any other purpose. */

extern malloc_mutex_t	chunks_mtx;

/* Chunk statistics. */

extern chunk_stats_t	stats_chunks;



extern rtree_t		*chunks_rtree;



extern size_t		chunksize;

extern size_t		chunksize_mask; /* (chunksize - 1). */

extern size_t		chunk_npages;

extern size_t		map_bias; /* Number of arena chunk header pages. */

extern size_t		arena_maxclass; /* Max size class for arenas. */



void	*chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,

    dss_prec_t dss_prec);

void	chunk_unmap(void *chunk, size_t size);

void	chunk_dealloc(void *chunk, size_t size, bool unmap);

bool	chunk_boot(void);

void	chunk_prefork(void);

void	chunk_postfork_parent(void);

void	chunk_postfork_child(void);



#endif /* JEMALLOC_H_EXTERNS */

/******************************************************************************/

#ifdef JEMALLOC_H_INLINES



#endif /* JEMALLOC_H_INLINES */

/******************************************************************************/



#include "jemalloc/internal/chunk_dss.h"

#include "jemalloc/internal/chunk_mmap.h"