Replaced Zend Memory Manager

Zend/zend.h

@@ -303,8 +303,10 @@ typedef enum {
 	 !(EG(current_execute_data)->prev_execute_data->opline->result_type & EXT_TYPE_UNUSED))
 
 #if defined(__GNUC__) && __GNUC__ >= 3 && !defined(__INTEL_COMPILER) && !defined(DARWIN) && !defined(__hpux) && !defined(_AIX) && !defined(__osf__)
+#  define ZEND_NORETURN __attribute__((noreturn))
 void zend_error_noreturn(int type, const char *format, ...) __attribute__ ((noreturn));
 #else
+#  define ZEND_NORETURN
 #  define zend_error_noreturn zend_error
 #endif
 

Zend/zend_alloc.h

@@ -26,19 +26,18 @@
 
 #include "../TSRM/TSRM.h"
 #include "zend.h"
-#include "zend_types.h"
 
 #ifndef ZEND_MM_ALIGNMENT
 # define ZEND_MM_ALIGNMENT Z_L(8)
 # define ZEND_MM_ALIGNMENT_LOG2 Z_L(3)
-#elif ZEND_MM_ALIGNMENT < Z_L(4)
+#elif ZEND_MM_ALIGNMENT < 4
 # undef ZEND_MM_ALIGNMENT
 # undef ZEND_MM_ALIGNMENT_LOG2
 # define ZEND_MM_ALIGNMENT Z_L(4)
 # define ZEND_MM_ALIGNMENT_LOG2 Z_L(2)
 #endif
 
-#define ZEND_MM_ALIGNMENT_MASK ~(ZEND_MM_ALIGNMENT-Z_L(1))
+#define ZEND_MM_ALIGNMENT_MASK ~(ZEND_MM_ALIGNMENT - Z_L(1))
 
 #define ZEND_MM_ALIGNED_SIZE(size)	(((size) + ZEND_MM_ALIGNMENT - Z_L(1)) & ZEND_MM_ALIGNMENT_MASK)
 
@@ -53,24 +52,98 @@ typedef struct _zend_leak_info {
 
 BEGIN_EXTERN_C()
 
-ZEND_API char *zend_strndup(const char *s, size_t length) ZEND_ATTRIBUTE_MALLOC;
+ZEND_API char*  ZEND_FASTCALL zend_strndup(const char *s, size_t length) ZEND_ATTRIBUTE_MALLOC;
 
-ZEND_API void *_emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE(1);
+ZEND_API void*  ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE(1);
-ZEND_API void *_safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
+ZEND_API void*  ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
-ZEND_API void *_safe_malloc(size_t nmemb, size_t size, size_t offset) ZEND_ATTRIBUTE_MALLOC;
+ZEND_API void*  ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset) ZEND_ATTRIBUTE_MALLOC;
-ZEND_API void _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ZEND_API void   ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
-ZEND_API void *_ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE2(1,2);
+ZEND_API void*  ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE2(1,2);
-ZEND_API void *_erealloc(void *ptr, size_t size, int allow_failure ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE(2);
+ZEND_API void*  ZEND_FASTCALL _erealloc(void *ptr, size_t size, int allow_failure ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_ALLOC_SIZE(2);
-ZEND_API void *_safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ZEND_API void*  ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
-ZEND_API void *_safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset);
+ZEND_API void*  ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset);
-ZEND_API char *_estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
+ZEND_API char*  ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
-ZEND_API char *_estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
+ZEND_API char*  ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) ZEND_ATTRIBUTE_MALLOC;
-ZEND_API size_t _zend_mem_block_size(void *ptr TSRMLS_DC ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr TSRMLS_DC ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
+
+#include "zend_alloc_sizes.h"
+
+/* _emalloc() & _efree() specialization */
+#if !ZEND_DEBUG && !defined(_WIN32)
+
+# define _ZEND_BIN_ALLOCATOR_DEF(_num, _size, _elements, _pages, x, y) \
+	ZEND_API void* ZEND_FASTCALL _emalloc_  ## _size(void) ZEND_ATTRIBUTE_MALLOC;
+
+ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR_DEF, x, y)
+
+ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE(1);
+ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size) ZEND_ATTRIBUTE_MALLOC ZEND_ATTRIBUTE_ALLOC_SIZE(1);
+
+# define _ZEND_BIN_ALLOCATOR_SELECTOR_START(_num, _size, _elements, _pages, size, y) \
+	((size <= _size) ? _emalloc_ ## _size() :
+# define _ZEND_BIN_ALLOCATOR_SELECTOR_END(_num, _size, _elements, _pages, size, y) \
+	)
+
+# define ZEND_ALLOCATOR(size) \
+	ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR_SELECTOR_START, size, y) \
+	((size <= ZEND_MM_MAX_LARGE_SIZE) ? _emalloc_large(size) : _emalloc_huge(size)) \
+	ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR_SELECTOR_END, size, y)
+
+# define _emalloc(size) \
+	(__builtin_constant_p(size) ? \
+		ZEND_ALLOCATOR(size) \
+	: \
+		_emalloc(size) \
+	)				
+
+# define _ZEND_BIN_DEALLOCATOR_DEF(_num, _size, _elements, _pages, x, y) \
+	ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *);
+
+ZEND_MM_BINS_INFO(_ZEND_BIN_DEALLOCATOR_DEF, x, y)
+
+ZEND_API void ZEND_FASTCALL _efree_large(void *, size_t size);
+ZEND_API void ZEND_FASTCALL _efree_huge(void *, size_t size);
+
+# define _ZEND_BIN_DEALLOCATOR_SELECTOR_START(_num, _size, _elements, _pages, ptr, size) \
+	if (size <= _size) { _efree_ ## _size(ptr); } else
+
+# define ZEND_DEALLOCATOR(ptr, size) \
+	ZEND_MM_BINS_INFO(_ZEND_BIN_DEALLOCATOR_SELECTOR_START, ptr, size) \
+	if (size <= ZEND_MM_MAX_LARGE_SIZE) { _efree_large(ptr, size); } \
+	else { _efree_huge(ptr, size); }
+
+# define efree_size(ptr, size) do { \
+		if (__builtin_constant_p(size)) { \
+			ZEND_DEALLOCATOR(ptr, size) \
+		} else { \
+			_efree(ptr); \
+		} \
+	} while (0)
+# define efree_size_rel(ptr, size) \
+	efree_size(ptr, size)
+
+#else
+
+# define efree_size(ptr, size) \
+	efree(ptr)
+# define efree_size_rel(ptr, size) \
+	efree_rel(ptr)
+
+#define _emalloc_large _emalloc
+#define _emalloc_huge  _emalloc
+#define _efree_large   _efree
+#define _efree_huge    _efree
+
+#endif
 
 /* Standard wrapper macros */
 #define emalloc(size)						_emalloc((size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
+#define emalloc_large(size)					_emalloc_large((size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
+#define emalloc_huge(size)					_emalloc_huge((size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 #define safe_emalloc(nmemb, size, offset)	_safe_emalloc((nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 #define efree(ptr)							_efree((ptr) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
+#define efree_large(ptr)					_efree_large((ptr) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
+#define efree_huge(ptr)						_efree_huge((ptr) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 #define ecalloc(nmemb, size)				_ecalloc((nmemb), (size) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 #define erealloc(ptr, size)					_erealloc((ptr), (size), 0 ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 #define safe_erealloc(ptr, nmemb, size, offset)	_safe_erealloc((ptr), (nmemb), (size), (offset) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
@@ -123,6 +196,7 @@ inline static void * __zend_realloc(void *p, size_t len)
 #define pemalloc(size, persistent) ((persistent)?__zend_malloc(size):emalloc(size))
 #define safe_pemalloc(nmemb, size, offset, persistent)	((persistent)?_safe_malloc(nmemb, size, offset):safe_emalloc(nmemb, size, offset))
 #define pefree(ptr, persistent)  ((persistent)?free(ptr):efree(ptr))
+#define pefree_size(ptr, size, persistent)  ((persistent)?free(ptr):efree_size(ptr, size))
 #define pecalloc(nmemb, size, persistent) ((persistent)?__zend_calloc((nmemb), (size)):ecalloc((nmemb), (size)))
 #define perealloc(ptr, size, persistent) ((persistent)?__zend_realloc((ptr), (size)):erealloc((ptr), (size)))
 #define safe_perealloc(ptr, nmemb, size, offset, persistent)	((persistent)?_safe_realloc((ptr), (nmemb), (size), (offset)):safe_erealloc((ptr), (nmemb), (size), (offset)))
@@ -143,17 +217,6 @@ ZEND_API void start_memory_manager(TSRMLS_D);
 ZEND_API void shutdown_memory_manager(int silent, int full_shutdown TSRMLS_DC);
 ZEND_API int is_zend_mm(TSRMLS_D);
 
-#if ZEND_DEBUG
-ZEND_API int _mem_block_check(void *ptr, int silent ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
-ZEND_API void _full_mem_check(int silent ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
-void zend_debug_alloc_output(char *format, ...);
-#define mem_block_check(ptr, silent) _mem_block_check(ptr, silent ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
-#define full_mem_check(silent) _full_mem_check(silent ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
-#else
-#define mem_block_check(type, ptr, silent)
-#define full_mem_check(silent)
-#endif
-
 ZEND_API size_t zend_memory_usage(int real_usage TSRMLS_DC);
 ZEND_API size_t zend_memory_peak_usage(int real_usage TSRMLS_DC);
 
@@ -164,13 +227,13 @@ END_EXTERN_C()
 	(ht) = (HashTable *) emalloc(sizeof(HashTable))
 
 #define FREE_HASHTABLE(ht)	\
-	efree(ht)
+	efree_size(ht, sizeof(HashTable))
 
 #define ALLOC_HASHTABLE_REL(ht)	\
 	(ht) = (HashTable *) emalloc_rel(sizeof(HashTable))
 
 #define FREE_HASHTABLE_REL(ht)	\
-	efree_rel(ht)
+	efree_size_rel(ht, sizeof(HashTable))
 
 /* Heap functions */
 typedef struct _zend_mm_heap zend_mm_heap;
@@ -192,32 +255,7 @@ ZEND_API size_t _zend_mm_block_size(zend_mm_heap *heap, void *p ZEND_FILE_LINE_D
 #define zend_mm_realloc_rel(heap, p, size)	_zend_mm_realloc((heap), (p), (size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_CC)
 #define zend_mm_block_size_rel(heap, p)		_zend_mm_block_size((heap), (p) ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC)
 
-/* Heaps with user defined storage */
-typedef struct _zend_mm_storage zend_mm_storage;
-
-typedef struct _zend_mm_segment {
-	size_t	size;
-	struct _zend_mm_segment *next_segment;
-} zend_mm_segment;
-
-typedef struct _zend_mm_mem_handlers {
-	const char *name;
-	zend_mm_storage* (*init)(void *params);
-	void (*dtor)(zend_mm_storage *storage);
-	void (*compact)(zend_mm_storage *storage);
-	zend_mm_segment* (*_alloc)(zend_mm_storage *storage, size_t size);
-	zend_mm_segment* (*_realloc)(zend_mm_storage *storage, zend_mm_segment *ptr, size_t size);
-	void (*_free)(zend_mm_storage *storage, zend_mm_segment *ptr);
-} zend_mm_mem_handlers;
-
-struct _zend_mm_storage {
-	const zend_mm_mem_handlers *handlers;
-	void *data;
-};
-
-ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_mem_handlers *handlers, size_t block_size, size_t reserve_size, int internal, void *params);
 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap TSRMLS_DC);
-ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap);
 
 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
                                           void* (*_malloc)(size_t),

Zend/zend_alloc_sizes.h

@@ -0,0 +1,46 @@
+#ifndef ZEND_ALLOC_SIZES_H
+#define ZEND_ALLOC_SIZES_H
+
+#define ZEND_MM_CHUNK_SIZE (2 * 1024 * 1024)               /* 2 MB  */
+#define ZEND_MM_PAGE_SIZE  (4 * 1024)                      /* 4 KB  */
+#define ZEND_MM_PAGES      (ZEND_MM_CHUNK_SIZE / ZEND_MM_PAGE_SIZE)  /* 512 */
+#define ZEND_MM_FIRST_PAGE (1)
+
+#define ZEND_MM_MIN_SMALL_SIZE		8
+#define ZEND_MM_MAX_SMALL_SIZE      3072
+#define ZEND_MM_MAX_LARGE_SIZE      (ZEND_MM_CHUNK_SIZE - (ZEND_MM_PAGE_SIZE * ZEND_MM_FIRST_PAGE))
+
+/* num, size, count, pages */
+#define ZEND_MM_BINS_INFO(_, x, y) \
+	_( 0,    8,  512, 1, x, y) \
+	_( 1,   16,  256, 1, x, y) \
+	_( 2,   24,  170, 1, x, y) \
+	_( 3,   32,  128, 1, x, y) \
+	_( 4,   40,  102, 1, x, y) \
+	_( 5,   48,   85, 1, x, y) \
+	_( 6,   56,   73, 1, x, y) \
+	_( 7,   64,   64, 1, x, y) \
+	_( 8,   80,   51, 1, x, y) \
+	_( 9,   96,   42, 1, x, y) \
+	_(10,  112,   36, 1, x, y) \
+	_(11,  128,   32, 1, x, y) \
+	_(12,  160,   25, 1, x, y) \
+	_(13,  192,   21, 1, x, y) \
+	_(14,  224,   18, 1, x, y) \
+	_(15,  256,   16, 1, x, y) \
+	_(16,  320,   64, 5, x, y) \
+	_(17,  384,   32, 3, x, y) \
+	_(18,  448,    9, 1, x, y) \
+	_(19,  512,    8, 1, x, y) \
+	_(20,  640,   32, 5, x, y) \
+	_(21,  768,   16, 3, x, y) \
+	_(22,  896,    9, 2, x, y) \
+	_(23, 1024,    8, 2, x, y) \
+	_(24, 1280,   16, 5, x, y) \
+	_(25, 1536,    8, 3, x, y) \
+	_(26, 1792,   16, 7, x, y) \
+	_(27, 2048,    8, 4, x, y) \
+	_(28, 2560,    8, 5, x, y) \
+	_(29, 3072,    4, 3, x, y)
+
+#endif /* ZEND_ALLOC_SIZES_H */

Zend/zend_variables.c

@@ -46,7 +46,7 @@ ZEND_API void _zval_dtor_func(zend_refcounted *p ZEND_FILE_LINE_DC)
 					GC_TYPE(arr) = IS_NULL;
 					GC_REMOVE_FROM_BUFFER(arr);
 					zend_hash_destroy(&arr->ht);
-					efree(arr);
+					efree_size(arr, sizeof(zend_array));
 				}
 				break;
 			}
@@ -54,7 +54,7 @@ ZEND_API void _zval_dtor_func(zend_refcounted *p ZEND_FILE_LINE_DC)
 				zend_ast_ref *ast = (zend_ast_ref*)p;
 		
 				zend_ast_destroy_and_free(ast->ast);
-				efree(ast);
+				efree_size(ast, sizeof(zend_ast_ref));
 				break;
 			}
 		case IS_OBJECT: {
@@ -78,7 +78,7 @@ ZEND_API void _zval_dtor_func(zend_refcounted *p ZEND_FILE_LINE_DC)
 				zend_reference *ref = (zend_reference*)p;
 				if (--GC_REFCOUNT(ref) == 0) {
 					zval_ptr_dtor(&ref->val);
-					efree(ref);
+					efree_size(ref, sizeof(zend_reference));
 				}
 				break;
 			}
@@ -106,7 +106,7 @@ ZEND_API void _zval_dtor_func_for_ptr(zend_refcounted *p ZEND_FILE_LINE_DC)
 					GC_TYPE(arr) = IS_NULL;
 					GC_REMOVE_FROM_BUFFER(arr);
 					zend_hash_destroy(&arr->ht);
-					efree(arr);
+					efree_size(arr, sizeof(zend_array));
 				}
 				break;
 			}
@@ -114,7 +114,7 @@ ZEND_API void _zval_dtor_func_for_ptr(zend_refcounted *p ZEND_FILE_LINE_DC)
 				zend_ast_ref *ast = (zend_ast_ref*)p;
 
 				zend_ast_destroy_and_free(ast->ast);
-				efree(ast);
+				efree_size(ast, sizeof(zend_ast_ref));
 				break;
 			}
 		case IS_OBJECT: {
@@ -136,7 +136,7 @@ ZEND_API void _zval_dtor_func_for_ptr(zend_refcounted *p ZEND_FILE_LINE_DC)
 				zend_reference *ref = (zend_reference*)p;
 
 				zval_ptr_dtor(&ref->val);
-				efree(ref);
+				efree_size(ref, sizeof(zend_reference));
 				break;
 			}
 		default: