archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-15 21:48:51 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Make distinct between indexes/numbers, GC addresses and pointers to gc_root_buffers.

Browse source 
Perform conversion through macros.
This commit is contained in:
Dmitry Stogov 2018-03-02 17:02:29 +03:00
parent 0fe7636084
commit ae64dd6d56
1 changed files with 116 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

212
Zend/zend_gc.c
View file

@ -119,24 +119,6 @@
GC_TYPE_INFO(ref) |= (GC_COLOR << GC_INFO_SHIFT); \
} while (0)
/* GC buffer size */
#define GC_INVALID 0
#define GC_FIRST_REAL_ROOT 1
#define GC_DEFAULT_BUF_SIZE (16 * 1024)
#define GC_BUF_GROW_STEP (128 * 1024)
#define GC_MAX_UNCOMPRESSED (1024 * 1024)
#define GC_MAX_BUF_SIZE 0x40000000
#define GC_THRESHOLD_DEFAULT 10000
#define GC_THRESHOLD_STEP 10000
#define GC_THRESHOLD_MAX 1000000000
#define GC_THRESHOLD_TRIGGER 100
/* GC flags */
#define GC_HAS_DESTRUCTORS (1<<0)
/* bit stealing tags for gc_root_buffer.ref */
#define GC_BITS 0x3
@ -157,9 +139,43 @@
#define GC_MAKE_GARBAGE(ptr) \
((void*)(((uintptr_t)(ptr)) | GC_GARBAGE))
/* GC address conversion */
#define GC_ADDR2NUM(addr) (addr)
#define GC_NUM2ADDR(num) (num)
#define GC_NEXT_ADDR(addr) ((addr) + 1)
#define GC_PREV_ADDR(addr) ((addr) - 1)
#define GC_ADDR2PTR(addr) (GC_G(buf) + (addr))
#define GC_PTR2ADDR(ptr) ((ptr) - GC_G(buf))
#define GC_ADDR2LIST(addr) ((void*)(uintptr_t)(((addr) * sizeof(void*)) | GC_UNUSED))
#define GC_LIST2ADDR(list) (((uint32_t)(uintptr_t)(list)) / sizeof(void*))
/* GC buffers */
#define GC_INVALID_NUM 0
#define GC_FIRST_ROOT_NUM 1
#define GC_INVALID_ADDR GC_NUM2ADDR(GC_INVALID_NUM)
#define GC_FIRST_ROOT_ADDR GC_NUM2ADDR(GC_FIRST_ROOT_NUM)
#define GC_DEFAULT_BUF_SIZE (16 * 1024)
#define GC_BUF_GROW_STEP (128 * 1024)
#define GC_MAX_UNCOMPRESSED (1024 * 1024)
#define GC_MAX_BUF_SIZE 0x40000000
#define GC_THRESHOLD_DEFAULT 10000
#define GC_THRESHOLD_STEP 10000
#define GC_THRESHOLD_MAX 1000000000
#define GC_THRESHOLD_TRIGGER 100
/* GC flags */
#define GC_HAS_DESTRUCTORS (1<<0)
/* unused buffers */
#define GC_HAS_UNUSED() \
(GC_G(unused) != GC_INVALID)
(GC_G(unused) != GC_INVALID_ADDR)
#define GC_FETCH_UNUSED() \
gc_fetch_unused()
#define GC_LINK_UNUSED(root) \
@ -185,10 +201,10 @@ typedef struct _zend_gc_globals {
zend_bool gc_full;
gc_root_buffer *buf; /* preallocated arrays of buffers */
uint32_t buf_size; /* size of the GC buffer */
uint32_t gc_threshold; /* GC collection threshold */
uint32_t unused; /* linked list of unused buffers */
uint32_t first_unused; /* first unused buffer */
uint32_t gc_threshold; /* GC collection threshold [addr] */
uint32_t buf_size; /* size of the GC buffer [addr] */
uint32_t num_roots; /* number of roots in GC buffer */
uint32_t gc_runs;
@ -248,7 +264,7 @@ static zend_always_inline uint32_t gc_compress(uint32_t addr)
static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t addr)
{
gc_root_buffer *root = GC_G(buf) + addr;
gc_root_buffer *root = GC_ADDR2PTR(addr);
if (EXPECTED(GC_GET_PTR(root->ref) == ref)) {
return root;
@ -257,7 +273,7 @@ static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, ui
while (1) {
addr += GC_MAX_UNCOMPRESSED;
ZEND_ASSERT(addr < GC_G(first_unused));
root = GC_G(buf) + addr;
root = GC_ADDR2PTR(addr);
if (GC_GET_PTR(root->ref) == ref) {
return root;
}
@ -271,17 +287,16 @@ static zend_always_inline uint32_t gc_fetch_unused(void)
ZEND_ASSERT(GC_HAS_UNUSED());
addr = GC_G(unused);
root = GC_G(buf) + addr;
root = GC_ADDR2PTR(addr);
ZEND_ASSERT(GC_IS_UNUSED(root->ref));
/* optimization: GC_GET_PTR(root->ref) is not necessary because it shifted anyway */
GC_G(unused) = (uint32_t)(uintptr_t)root->ref / sizeof(void*);
GC_G(unused) = GC_LIST2ADDR(root->ref);
return addr;
}
static zend_always_inline void gc_link_unused(gc_root_buffer *root)
{
root->ref = (void*)(uintptr_t)((GC_G(unused) * sizeof(void*)) | GC_UNUSED);
GC_G(unused) = root - GC_G(buf);
root->ref = GC_ADDR2LIST(GC_G(unused));
GC_G(unused) = GC_PTR2ADDR(root);
}
static zend_always_inline uint32_t gc_fetch_next_unused(void)
@ -290,7 +305,7 @@ static zend_always_inline uint32_t gc_fetch_next_unused(void)
ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
addr = GC_G(first_unused);
GC_G(first_unused)++;
GC_G(first_unused) = GC_NEXT_ADDR(GC_G(first_unused));
return addr;
}
@ -349,10 +364,10 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals)
gc_globals->gc_full = 0;
gc_globals->buf = NULL;
gc_globals->buf_size = 0;
gc_globals->gc_threshold = 0;
gc_globals->unused = GC_INVALID;
gc_globals->first_unused = 0;
gc_globals->unused = GC_INVALID_ADDR;
gc_globals->first_unused = GC_INVALID_ADDR;
gc_globals->gc_threshold = GC_INVALID_ADDR;
gc_globals->buf_size = GC_INVALID_ADDR;
gc_globals->num_roots = 0;
gc_globals->gc_runs = 0;
@ -390,8 +405,8 @@ void gc_reset(void)
GC_G(gc_active) = 0;
GC_G(gc_protected) = 0;
GC_G(gc_full) = 0;
GC_G(unused) = GC_INVALID;
GC_G(first_unused) = GC_FIRST_REAL_ROOT;
GC_G(unused) = GC_INVALID_ADDR;
GC_G(first_unused) = GC_FIRST_ROOT_ADDR;
GC_G(num_roots) = 0;
GC_G(gc_runs) = 0;
@ -414,8 +429,8 @@ ZEND_API zend_bool gc_enable(zend_bool enable)
GC_G(gc_enabled) = enable;
if (enable && !old_enabled && GC_G(buf) == NULL) {
GC_G(buf) = (gc_root_buffer*) malloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE);
GC_G(buf_size) = GC_DEFAULT_BUF_SIZE;
GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT + GC_FIRST_REAL_ROOT;
GC_G(buf_size) = GC_NUM2ADDR(GC_DEFAULT_BUF_SIZE);
GC_G(gc_threshold) = GC_NUM2ADDR(GC_THRESHOLD_DEFAULT + GC_FIRST_ROOT_NUM);
gc_reset();
}
return old_enabled;
@ -442,7 +457,7 @@ static void gc_grow_root_buffer(void)
{
size_t new_size;
if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) {
if (GC_ADDR2NUM(GC_G(buf_size)) >= GC_MAX_BUF_SIZE) {
if (!GC_G(gc_full)) {
zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n");
GC_G(gc_active) = 1;
@ -451,16 +466,16 @@ static void gc_grow_root_buffer(void)
return;
}
}
if (GC_G(buf_size) < GC_BUF_GROW_STEP) {
new_size = GC_G(buf_size) * 2;
if (GC_G(buf_size) < GC_NUM2ADDR(GC_BUF_GROW_STEP)) {
new_size = GC_ADDR2NUM(GC_G(buf_size)) * 2;
} else {
new_size = GC_G(buf_size) + GC_BUF_GROW_STEP;
new_size = GC_ADDR2NUM(GC_G(buf_size)) + GC_BUF_GROW_STEP;
}
if (new_size > GC_MAX_BUF_SIZE) {
new_size = GC_MAX_BUF_SIZE;
}
GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1);
GC_G(buf_size) = new_size;
GC_G(buf_size) = GC_NUM2ADDR(new_size);
}
static void gc_adjust_threshold(int count)
@ -472,24 +487,24 @@ static void gc_adjust_threshold(int count)
* by a fixed step */
if (count < GC_THRESHOLD_TRIGGER) {
/* increase */
if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) {
new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP;
if (GC_G(gc_threshold) < GC_NUM2ADDR(GC_THRESHOLD_MAX)) {
new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) + GC_THRESHOLD_STEP;
if (new_threshold > GC_THRESHOLD_MAX) {
new_threshold = GC_THRESHOLD_MAX;
}
if (new_threshold > GC_G(buf_size)) {
if (new_threshold > GC_ADDR2NUM(GC_G(buf_size))) {
gc_grow_root_buffer();
}
if (new_threshold <= GC_G(buf_size)) {
GC_G(gc_threshold) = new_threshold;
if (new_threshold <= GC_ADDR2NUM(GC_G(buf_size))) {
GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold);
}
}
} else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) {
new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP;
} else if (GC_G(gc_threshold) > GC_NUM2ADDR(GC_THRESHOLD_DEFAULT)) {
new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) - GC_THRESHOLD_STEP;
if (new_threshold < GC_THRESHOLD_DEFAULT) {
new_threshold = GC_THRESHOLD_DEFAULT;
}
GC_G(gc_threshold) = new_threshold;
GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold);
}
}
@ -518,11 +533,13 @@ static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refc
addr = GC_FETCH_NEXT_UNUSED();
} else {
gc_grow_root_buffer();
ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
return;
}
addr = GC_FETCH_NEXT_UNUSED();
}
newRoot = GC_G(buf) + addr;
newRoot = GC_ADDR2PTR(addr);
newRoot->ref = ref; /* GC_ROOT tag is 0 */
GC_TRACE_SET_COLOR(ref, GC_PURPLE);
@ -558,7 +575,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref)
ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
ZEND_ASSERT(GC_INFO(ref) == 0);
newRoot = GC_G(buf) + addr;
newRoot = GC_ADDR2PTR(addr);
newRoot->ref = ref; /* GC_ROOT tag is 0 */
GC_TRACE_SET_COLOR(ref, GC_PURPLE);
@ -592,12 +609,12 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
GC_REF_SET_INFO(ref, 0);
/* Perform decopression only in case of large buffers */
if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) {
if (UNEXPECTED(GC_G(first_unused) >= GC_NUM2ADDR(GC_MAX_UNCOMPRESSED))) {
gc_remove_compressed(ref, addr);
return;
}
root = GC_G(buf) + addr;
root = GC_ADDR2PTR(addr);
gc_remove_from_roots(root);
}
@ -813,30 +830,30 @@ tail_call:
/* Two-Finger compaction algorithm */
static void gc_compact(void)
{
if (GC_G(num_roots) + GC_FIRST_REAL_ROOT != GC_G(first_unused)) {
if (GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM) != GC_G(first_unused)) {
if (GC_G(num_roots)) {
gc_root_buffer *buf = GC_G(buf);
uint32_t free = GC_FIRST_REAL_ROOT;
uint32_t scan = GC_G(first_unused) - 1;
gc_root_buffer *free = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
gc_root_buffer *scan = GC_ADDR2PTR(GC_PREV_ADDR(GC_G(first_unused)));
gc_root_buffer *end = GC_ADDR2PTR(GC_NUM2ADDR(GC_G(num_roots)));
uint32_t addr;
zend_refcounted *p;
while (free < scan) {
while (!GC_IS_UNUSED(buf[free].ref)) {
while (!GC_IS_UNUSED(free->ref)) {
free++;
}
while (GC_IS_UNUSED(buf[scan].ref)) {
while (GC_IS_UNUSED(scan->ref)) {
scan--;
}
if (scan > free) {
p = buf[scan].ref;
buf[free].ref = p;
p = scan->ref;
free->ref = p;
p = GC_GET_PTR(p);
addr = gc_compress(free);
addr = gc_compress(GC_PTR2ADDR(free));
GC_REF_SET_INFO(p, addr | GC_REF_COLOR(p));
free++;
scan--;
if (scan <= GC_G(num_roots)) {
if (scan <= end) {
break;
}
} else {
@ -844,8 +861,8 @@ static void gc_compact(void)
}
}
}
GC_G(unused) = GC_INVALID;
GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_REAL_ROOT;
GC_G(unused) = GC_INVALID_ADDR;
GC_G(first_unused) = GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM);
}
}
@ -855,8 +872,8 @@ static void gc_mark_roots(void)
gc_compact();
current = GC_G(buf) + GC_FIRST_REAL_ROOT;
last = GC_G(buf) + GC_G(first_unused);
current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);;
last = GC_ADDR2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_ROOT(current->ref)) {
if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) {
@ -966,8 +983,8 @@ tail_call:
static void gc_scan_roots(void)
{
gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT;
gc_root_buffer *last = GC_G(buf) + GC_G(first_unused);
gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_ROOT(current->ref)) {
@ -988,11 +1005,13 @@ static void gc_add_garbage(zend_refcounted *ref)
addr = GC_FETCH_NEXT_UNUSED();
} else {
gc_grow_root_buffer();
ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
return;
}
addr = GC_FETCH_NEXT_UNUSED();
}
buf = GC_G(buf) + addr;
buf = GC_ADDR2PTR(addr);
buf->ref = GC_MAKE_GARBAGE(ref);
addr = gc_compress(addr);
@ -1131,11 +1150,11 @@ tail_call:
static int gc_collect_roots(uint32_t *flags)
{
uint32_t n, end;
uint32_t addr, end;
zend_refcounted *ref;
int count = 0;
gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT;
gc_root_buffer *last = GC_G(buf) + GC_G(first_unused);
gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused));
/* remove non-garbage from the list */
while (current != last) {
@ -1152,10 +1171,10 @@ static int gc_collect_roots(uint32_t *flags)
/* Root buffer might be reallocated during gc_collect_white,
* make sure to reload pointers. */
n = GC_FIRST_REAL_ROOT;
addr = GC_FIRST_ROOT_ADDR;
end = GC_G(first_unused);
while (n != end) {
current = GC_G(buf) + n;
while (addr != end) {
current = GC_ADDR2PTR(addr);
ref = current->ref;
if (GC_IS_ROOT(ref)) {
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
@ -1163,7 +1182,7 @@ static int gc_collect_roots(uint32_t *flags)
count += gc_collect_white(ref, flags);
}
}
n++;
addr = GC_NEXT_ADDR(addr);
}
return count;
@ -1275,7 +1294,7 @@ ZEND_API int zend_gc_collect_cycles(void)
gc_root_buffer *current, *last;
zend_refcounted *p;
uint32_t gc_flags = 0;
uint32_t n, end;
uint32_t addr, end;
if (GC_G(gc_active)) {
return 0;
@ -1303,17 +1322,18 @@ ZEND_API int zend_gc_collect_cycles(void)
end = GC_G(first_unused);
if (gc_flags & GC_HAS_DESTRUCTORS) {
uint32_t *refcounts;
uint32_t *refcounts, count, n;
GC_TRACE("Calling destructors");
// TODO: may be use emalloc() ???
refcounts = pemalloc(sizeof(uint32_t) * GC_G(first_unused), 1);
count = GC_ADDR2NUM(GC_G(first_unused));
refcounts = pemalloc(sizeof(uint32_t) * count, 1);
/* Remember reference counters before calling destructors */
n = GC_FIRST_REAL_ROOT;
current = GC_G(buf) + GC_FIRST_REAL_ROOT;
while (n != end) {
n = GC_FIRST_ROOT_NUM;
current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
while (n != count) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
refcounts[n] = GC_REFCOUNT(p);
@ -1326,9 +1346,9 @@ ZEND_API int zend_gc_collect_cycles(void)
*
* The root buffer might be reallocated during destructors calls,
* make sure to reload pointers as necessary. */
n = GC_FIRST_REAL_ROOT;
while (n != end) {
current = GC_G(buf) + n;
addr = GC_FIRST_ROOT_ADDR;
while (addr != end) {
current = GC_ADDR2PTR(addr);
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
if (GC_TYPE(p) == IS_OBJECT
@ -1346,13 +1366,13 @@ ZEND_API int zend_gc_collect_cycles(void)
}
}
}
n++;
addr = GC_NEXT_ADDR(addr);
}
/* Remove values captured in destructors */
n = GC_FIRST_REAL_ROOT;
current = GC_G(buf) + GC_FIRST_REAL_ROOT;
while (n != end) {
n = GC_FIRST_ROOT_NUM;
current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
while (n != count) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
if (GC_REFCOUNT(p) > refcounts[n]) {
@ -1374,8 +1394,8 @@ ZEND_API int zend_gc_collect_cycles(void)
/* Destroy zvals */
GC_TRACE("Destroying zvals");
GC_G(gc_protected) = 1;
current = GC_G(buf) + GC_FIRST_REAL_ROOT;
last = GC_G(buf) + GC_G(first_unused);
current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
last = GC_ADDR2PTR(GC_G(first_unused));
while (current != last) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);
@ -1413,7 +1433,7 @@ ZEND_API int zend_gc_collect_cycles(void)
}
/* Free objects */
current = GC_G(buf) + GC_FIRST_REAL_ROOT;
current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
while (current != last) {
if (GC_IS_GARBAGE(current->ref)) {
p = GC_GET_PTR(current->ref);