diff --git a/include/lighttpd/mempool.h b/include/lighttpd/mempool.h new file mode 100644 index 0000000..030c096 --- /dev/null +++ b/include/lighttpd/mempool.h @@ -0,0 +1,20 @@ +#ifndef _LIGHTTPD_MEMPOOL_H_ +#define _LIGHTTPD_MEMPOOL_H_ + +#include + +typedef struct mempool_ptr mempool_ptr; +struct mempool_ptr { + void *priv_data; /* private data for internal management */ + void *data; /* real pointer (result of alloc) */ +}; + +LI_API gsize mempool_align_page_size(gsize size); +LI_API mempool_ptr mempool_alloc(gsize size); + +/* you cannot release parts from an allocated chunk; so you _have_ to remember the size from the alloc */ +LI_API void mempool_free(mempool_ptr ptr, gsize size); + +LI_API void mempool_cleanup(); + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e2e0256..d57790e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -157,6 +157,7 @@ SET(COMMON_SRC encoding.c idlist.c ip_parsers.c + mempool.c module.c radix.c sys-files.c diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 9341a25..e5b8d6f 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -8,6 +8,7 @@ common_src= \ encoding.c \ idlist.c \ ip_parsers.c \ + mempool.c \ module.c \ radix.c \ sys-files.c \ diff --git a/src/common/mempool.c b/src/common/mempool.c new file mode 100644 index 0000000..95cdf73 --- /dev/null +++ b/src/common/mempool.c @@ -0,0 +1,477 @@ + +#include + +/* + * available #defines: + * - MEMPOOL_MALLOC + * just use malloc, "disable" mempool + * - MEMPOOL_SAFE_MUTEX + * use g_mutex in mempool to synchronize access in magazines; + * by default we use spinlocks (busy wait), as we assume that memory + * objects stay in one thread by default + * - MP_SEARCH_BITVECTOR + * search in bitvector of a magazine for free chunks + */ + +#define MP_SEARCH_BITVECTOR + + +#ifdef MEMPOOL_MALLOC + +mempool_ptr mempool_alloc(gsize size) { + mempool_ptr ptr = { NULL, g_malloc(size) }; + return ptr; +} + +void mempool_free(mempool_ptr ptr, gsize size) { + if (!ptr.data) return; + free(ptr.data); +} + +void mempool_cleanup() { +} + +#else /* MP_MALLOC */ + +/* + * mempool: + * - allocate memory for a chunk; each thread has up to 2 magazines for each chunk size we use (page-aligned); + * each magazine has one mmap-area from which chunks are allocated + * - if MAP_ANON is not available (for mmap) use malloc instead to allocate the magazine area; as the size of + * these areas exceeds 1MB perhaps the default malloc() uses a sane fallback... (instead of brk()). + * - if a magazine is full, the thread allocates a new one; magazines aren't reused + * - if MP_SEARCH_BITVECTOR is defined, we search for free chunks in the bitvector; + * if not, we don't even reuse chunks in a "active" magazine, unless it is the last one we allocated from it + * - needed characteristics are: + * + fast for few sizes (all page-aligned) + * + allow complex interface (don't want to replace malloc/free) + * + assume memory is going to be released soon again + * + avoid fragmentation with other allocators (like 1024*malloc(16kb); malloc(1byte); 1024*free(16kb) - 16mb "garbage") + */ + +# define UL_BITS (sizeof(gulong) * 8) + +# define MP_MAX_ALLOC_SIZE (8*1024*1024) +# define MP_MIN_ALLOC_COUNT 8 +# define MP_MAX_ALLOC_COUNT 256 +# define MP_MAX_MAGAZINES 2 + +# define MP_BIT_VECTOR_SIZE ((MP_MAX_ALLOC_COUNT + UL_BITS - 1)/UL_BITS) + +# if 0 +# define mp_assert(x) g_assert(x) +# else +# define mp_assert(x) do { (void) 0; } while (0) +# endif + + +# ifdef MEMPOOL_SAFE_MUTEX +/* use GMutex */ +typedef GMutex* mp_lock; +# define MP_LOCK_NEW() g_mutex_new() +# define MP_LOCK_FREE(lock) g_mutex_free(lock) +# define MP_LOCK(lock) g_mutex_lock(lock) +# define MP_TRYLOCK(lock) g_mutex_trylock(lock) +# define MP_UNLOCK(lock) g_mutex_unlock(lock) +# else +/* use spinlocks */ +typedef gint mp_lock; +# define MP_LOCK_NEW() (1) +# define MP_LOCK_FREE(lock) do { (void) 0; } while (0) +# define MP_LOCK(lock) do { (void) 0; } while (!g_atomic_int_compare_and_exchange(&lock, 1, 0)) +# define MP_TRYLOCK(lock) (g_atomic_int_compare_and_exchange(&lock, 1, 0)) +# define MP_UNLOCK(lock) (g_atomic_int_set(&lock, 1)) +# endif + + +typedef struct mp_pools mp_pools; +typedef struct mp_pool mp_pool; +typedef struct mp_magazine mp_magazine; + +struct mp_pool { + guint32 chunksize; + + /* if magazines[i+1] != NULL => magazines[i] != NULL - only the "head" entries are not NULL */ + /* so we can stop searching if an entry is NULL */ + mp_magazine *magazines[MP_MAX_MAGAZINES]; + + GList pools_list; /* list element for the mp_pools.queue */ +}; + +struct mp_magazine { + gint refcount; /* one ref from pool + one per allocated chunk */ + void *data; /* pointer to mmap area */ + guint32 chunksize; + guint32 used, count; +# ifndef MP_SEARCH_BITVECTOR + guint32 next; +# endif + gulong bv_used[MP_BIT_VECTOR_SIZE]; + + mp_lock mutex; +}; + +/* one queue of pools per thread */ +struct mp_pools { + /* one pool per chunksize; queue is sorted ASC by chunksize */ + GQueue queue; +}; + +static void mp_pools_free(gpointer _pools); + +static GPrivate *thread_pools = NULL; +static gboolean mp_initialized = 0; +static gsize mp_pagesize = 0; + +static GStaticMutex mp_init_mutex = G_STATIC_MUTEX_INIT; + +static void mempool_init() { + g_static_mutex_lock (&mp_init_mutex); + if (!mp_initialized) { + mp_pagesize = sysconf(_SC_PAGE_SIZE); + if (0 == mp_pagesize) mp_pagesize = 4*1024; /* 4kb default */ + + if (!g_thread_supported()) g_thread_init (NULL); + + thread_pools = g_private_new(mp_pools_free); + + mp_initialized = TRUE; + } + g_static_mutex_unlock (&mp_init_mutex); +} + +/* only call if you know that mempool_init was called */ +static inline gsize mp_align_size(gsize size) { + /* assume pagesize is 2^n */ + size = (size + mp_pagesize - 1) & ~(mp_pagesize-1); + return size; +} + +gsize mempool_align_page_size(gsize size) { + if (G_UNLIKELY(!mp_initialized)) { + mempool_init(); + } + return mp_align_size(size); +} + +static inline void* mp_alloc_page(gsize size) { + void *ptr; +# ifdef MAP_ANON + if (G_UNLIKELY(MAP_FAILED == (ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0)))) { + g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes with mmap", G_STRLOC, size); + } +# else + if (G_UNLIKELY(NULL == (ptr = malloc(size)))) { + g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes", G_STRLOC, size); + } +# endif + return ptr; +} + +static inline void mp_free_page(const void *ptr, gsize size) { + if (!ptr) return; +# ifdef MAP_ANON + munmap((void*) ptr, size); +# else + free(ptr); +# endif +} + +/* how many chunks in an arey for a chunk size? */ +static gsize mp_chunks_for_size(gsize size) { + gsize chunks = MP_MAX_ALLOC_SIZE/size; + if (chunks > MP_MAX_ALLOC_COUNT) chunks = MP_MAX_ALLOC_COUNT; + return chunks; +} + +static mp_magazine* mp_mag_new(mp_pool *pool) { + mp_magazine *mag = g_slice_new0(mp_magazine); + mag->refcount = 1; + mag->chunksize = pool->chunksize; + mag->used = 0; +# ifndef MP_SEARCH_BITVECTOR + mag->next = 0; +# endif + mag->count = mp_chunks_for_size(mag->chunksize); + mag->mutex = MP_LOCK_NEW(); + + return mag; +} + +/* do NOT call with lock held */ +static inline void mp_mag_release(mp_magazine *mag) { + if (!mag) return; + assert(g_atomic_int_get(&mag->refcount) > 0); + if (g_atomic_int_dec_and_test(&mag->refcount)) { + MP_LOCK_FREE(mag->mutex); + g_slice_free(mp_magazine, mag); + } +} + +static inline void mp_mag_acquire(mp_magazine *mag) { + assert(g_atomic_int_get(&mag->refcount) > 0); + g_atomic_int_inc(&mag->refcount); +} + +static inline guint find_free_bit(gulong l) { + guint idx; + guint8 b; + + mp_assert(~0ul != l); + + for (idx = 0; ((guint8)~0u) == (b = (guint8)(l >> idx)); idx += 8) ; + for ( ; (b & 1); b >>= 1, idx++ ) ; + + return idx; +} + +/* call with lock held */ +static inline void* mp_mag_alloc(mp_magazine *mag) { + gulong *bv = mag->bv_used; +# ifndef MP_SEARCH_BITVECTOR + guint id = mag->next, bndx = id % UL_BITS, ndx = id / UL_BITS; +# else + guint id, ndx; +# endif + + mp_assert(mag->used < mag->count); + mp_assert(mag->next < mag->count); + + if (NULL == mag->data) { + mag->data = mp_alloc_page(mag->count * mag->chunksize); + } + +# ifndef MP_SEARCH_BITVECTOR + bv[ndx] |= (1ul << bndx); + mag->next++; +# else + for (ndx = 0; ndx < MP_BIT_VECTOR_SIZE && (bv[ndx] == ~0ul); ndx++) ; + mp_assert(ndx < MP_BIT_VECTOR_SIZE); + + id = find_free_bit(bv[ndx]); + /* id = g_bit_nth_lsf(~bv[ndx], -1); */ + bv[ndx] |= (1ul << id); + id += UL_BITS * ndx; +# endif + + mag->used++; + + return (void*) (((intptr_t)mag->data) + (id * mag->chunksize)); +} + +/* call with lock held; update ref counter after releasing lock! */ +static inline void mp_mag_free(mp_magazine *mag, void *ptr) { + guint id = (((intptr_t) ptr) - ((intptr_t) mag->data)) / mag->chunksize; + guint ndx = id / UL_BITS, bndx = id % UL_BITS; + gulong bmask = 1ul << bndx; + + mp_assert(ndx < MP_BIT_VECTOR_SIZE); + g_assert(0 != (mag->bv_used[ndx] & bmask)); /* check for double free */ + + mag->bv_used[ndx] &= ~bmask; + mag->used--; + +# ifndef MP_SEARCH_BITVECTOR + if (id == mag->next - 1) mag->next--; /* if chunk was just allocated, "undo" it */ +# endif + + if (G_UNLIKELY(0 == mag->used)) { + mp_free_page(mag->data, mag->count * mag->chunksize); + mag->data = NULL; +# ifndef MP_SEARCH_BITVECTOR + mag->next = 0; +# endif + } +} + +static mp_pool* mp_pool_new(gsize size) { + mp_pool *pool = g_slice_new0(mp_pool); + pool->chunksize = size; + pool->pools_list.data = pool; + pool->magazines[0] = mp_mag_new(pool); + + return pool; +} + +static void mp_pool_free(mp_pool *pool) { + guint i; + mp_magazine *mag; + if (!pool) return; + + for (i = 0; i < MP_MAX_MAGAZINES; i++) { + mag = pool->magazines[i]; + pool->magazines[i] = NULL; + mp_mag_release(mag); + } + + g_slice_free(mp_pool, pool); +} + +static void _queue_insert_before(GQueue *queue, GList *sibling, GList *item) { + item->prev = sibling->prev; + item->next = sibling; + sibling->prev = item; + if (NULL == item->prev) { + queue->head = item; + } else { + item->prev->next = item; + } + queue->length++; +} + +static void mp_pools_free(gpointer _pools) { + mp_pools *pools = _pools; + mp_pool *pool; + GList *iter; + + while (NULL != (iter = g_queue_pop_head_link(&pools->queue))) { + pool = iter->data; + + mp_pool_free(pool); + } + + g_slice_free(mp_pools, pools); +} + +static inline mp_pool* mp_pools_get(gsize size) { + GList *iter; + mp_pools *pools; + mp_pool *pool; + + pools = g_private_get(thread_pools); + if (G_UNLIKELY(!pools)) { + pools = g_slice_new0(mp_pools); + g_private_set(thread_pools, pools); + } + + for (iter = pools->queue.head; iter; iter = iter->next) { + pool = iter->data; + if (G_LIKELY(pool->chunksize == size)) { + goto done; + } else if (G_UNLIKELY(pool->chunksize > size)) { + pool = mp_pool_new(size); + _queue_insert_before(&pools->queue, iter, &pool->pools_list); + goto done; + } + } + + pool = mp_pool_new(size); + g_queue_push_tail_link(&pools->queue, &pool->pools_list); + +done: + return pool; +} + +mempool_ptr mempool_alloc(gsize size) { + mempool_ptr ptr = { NULL, NULL }; + mp_pool *pool; + mp_magazine *mag; + guint i; + + if (G_UNLIKELY(!mp_initialized)) { + mempool_init(); + } + + size = mp_align_size(size); + + /* mp_alloc_page fallback */ + if (G_UNLIKELY(size > MP_MAX_ALLOC_SIZE/MP_MIN_ALLOC_COUNT)) { + if (G_UNLIKELY(NULL == (ptr.data = mp_alloc_page(size)))) { + g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes", G_STRLOC, size); + } + return ptr; + } + + pool = mp_pools_get(size); + + /* Try to lock a unlocked magazine if possible; creating new magazines is allowed + * (new ones can't be locked as only the current thread knows this magazine) + * Spinlock the first magazine if first strategy failed */ + if (G_LIKELY(pool->magazines[0])) { + /* at least one magazine available */ + for (i = 0; i < MP_MAX_MAGAZINES; i++) { + mag = pool->magazines[i]; + if (!mag) break; + if (G_LIKELY(MP_TRYLOCK(mag->mutex))) { + goto found_mag; + } + } + i = 0; + mag = pool->magazines[0]; + MP_LOCK(mag->mutex); + } else { + /* no magazine - just create one */ + i = 0; + mag = pool->magazines[0] = mp_mag_new(pool); + MP_LOCK(mag->mutex); + } +found_mag: + + ptr.priv_data = mag; + ptr.data = mp_mag_alloc(mag); + mp_mag_acquire(mag); /* keep track of chunk count */ + +# ifndef MP_SEARCH_BITVECTOR + if (G_UNLIKELY(mag->next == mag->count)) { +# else + if (G_UNLIKELY(mag->used == mag->count)) { +# endif + /* full magazine; remove from pool */ + guint j; + + /* replace entry with last entry != NULL */ + for (j = i+1; j < MP_MAX_MAGAZINES && pool->magazines[j]; j++) ; + if (j < MP_MAX_MAGAZINES) { + pool->magazines[i] = pool->magazines[j]; + pool->magazines[j] = NULL; + } else { + pool->magazines[i] = NULL; + } + + MP_UNLOCK(mag->mutex); + mp_mag_release(mag); /* keep track of pool -> magazine ref; release always after unlock! */ + } else { + MP_UNLOCK(mag->mutex); + } + + return ptr; +} + +void mempool_free(mempool_ptr ptr, gsize size) { + mp_magazine *mag; + if (!ptr.data) return; + + size = mp_align_size(size); + + /* mp_alloc_page fallback */ + if (G_UNLIKELY(size > MP_MAX_ALLOC_SIZE/MP_MIN_ALLOC_COUNT)) { + mp_free_page(ptr.data, size); + return; + } + + mp_assert(ptr.priv_data); + mag = ptr.priv_data; + MP_LOCK(mag->mutex); + mp_mag_free(mag, ptr.data); + MP_UNLOCK(mag->mutex); + + mp_mag_release(mag); /* keep track of chunk count; release always after unlock! */ +} + +void mempool_cleanup() { + /* "Force" thread-private cleanup */ + mp_pools *pools; + + if (G_UNLIKELY(!mp_initialized)) { + mempool_init(); + } + + pools = g_private_get(thread_pools); + + if (pools) { + g_private_set(thread_pools, NULL); + mp_pools_free(pools); + } +} + +#endif /* !MP_MALLOC */ diff --git a/src/common/wscript b/src/common/wscript index b35d1b8..0f11573 100644 --- a/src/common/wscript +++ b/src/common/wscript @@ -20,6 +20,7 @@ def build(bld): encoding.c idlist.c ip_parsers.rl + mempool.c module.c radix.c utils.c diff --git a/src/main/lighttpd.c b/src/main/lighttpd.c index de4721f..7083e49 100644 --- a/src/main/lighttpd.c +++ b/src/main/lighttpd.c @@ -168,5 +168,7 @@ int main(int argc, char *argv[]) { if (free_config_path) g_free(config_path); + mempool_cleanup(); + return 0; }