archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-15 21:48:51 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refactor GC data traversal loops

Browse source 
Previusly, to redce stack usage GC traversed object properties and array
elements in both directions. At first, it seeked backward from the last
element, to find the last refcounted zval. Then, it traversed elements
forward. As result, we had 2 non-spatial memory accesses for each
object/array and usuall 2 expensive CPU cache misses.

Now all the traversal algorithms are refactored to scan elements only in
forward direction. The number of checks and GC stack usage should be the
same. The order of element processing may be differenr, but this
should not be a problem.
This commit is contained in:
Dmitry Stogov 2021-11-24 20:38:00 +03:00
parent 885f935090
commit 4b4b9edcc0
1 changed files with 416 additions and 506 deletions
Download patch file Download diff file Expand all files Collapse all files

666
Zend/zend_gc.c
View file

@ -691,8 +691,10 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
static void gc_scan_black(zend_refcounted *ref, gc_stack *stack) static void gc_scan_black(zend_refcounted *ref, gc_stack *stack)
{ {
HashTable *ht = NULL; HashTable *ht;
Bucket *p;
zval *zv; zval *zv;
uint32_t n;
GC_STACK_DCL(stack); GC_STACK_DCL(stack);
tail_call: tail_call:
@ -700,29 +702,17 @@ tail_call:
zend_object *obj = (zend_object*)ref; zend_object *obj = (zend_object*)ref;
if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
int n; zval *table;
zval *zv, *end; int len;
ht = obj->handlers->get_gc(obj, &zv, &n); ht = obj->handlers->get_gc(obj, &table, &len);
n = len;
zv = table;
if (UNEXPECTED(ht)) { if (UNEXPECTED(ht)) {
GC_ADDREF(ht); GC_ADDREF(ht);
if (!GC_REF_CHECK_COLOR(ht, GC_BLACK)) { if (!GC_REF_CHECK_COLOR(ht, GC_BLACK)) {
GC_REF_SET_BLACK(ht); GC_REF_SET_BLACK(ht);
} else { for (; n != 0; n--) {
ht = NULL;
}
}
if (EXPECTED(!ht)) {
if (!n) goto next;
end = zv + n;
while (!Z_REFCOUNTED_P(--end)) {
if (zv == end) goto next;
}
} else {
if (!n) goto handle_ht;
end = zv + n;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_ADDREF(ref); GC_ADDREF(ref);
@ -733,21 +723,77 @@ tail_call:
} }
zv++; zv++;
} }
if (EXPECTED(!ht)) { goto handle_ht;
}
}
handle_zvals:
for (; n != 0; n--) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_ADDREF(ref); GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref); GC_REF_SET_BLACK(ref);
zv++;
while (--n) {
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
}
}
zv++;
}
goto tail_call; goto tail_call;
} }
goto next;
} }
} else { zv++;
goto next; }
} }
} else if (GC_TYPE(ref) == IS_ARRAY) { } else if (GC_TYPE(ref) == IS_ARRAY) {
ZEND_ASSERT((zend_array*)ref != &EG(symbol_table)); ZEND_ASSERT((zend_array*)ref != &EG(symbol_table));
ht = (zend_array*)ref; ht = (zend_array*)ref;
handle_ht:
n = ht->nNumUsed;
zv = ht->arPacked;
if (HT_IS_PACKED(ht)) {
goto handle_zvals;
}
p = (Bucket*)zv;
for (; n != 0; n--) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
p++;
while (--n) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
}
}
p++;
}
goto tail_call;
}
}
p++;
}
} else if (GC_TYPE(ref) == IS_REFERENCE) { } else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) { if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val); ref = Z_COUNTED(((zend_reference*)ref)->val);
@ -757,79 +803,8 @@ tail_call:
goto tail_call; goto tail_call;
} }
} }
goto next;
} else {
goto next;
} }
handle_ht:
if (!ht->nNumUsed) goto next;
if (HT_IS_PACKED(ht)) {
zval *end;
zv = ht->arPacked;
end = zv + ht->nNumUsed;
while (1) {
end--;
if (Z_REFCOUNTED_P(end)) {
break;
}
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
}
}
zv++;
}
} else {
Bucket *p = ht->arData;
Bucket *end = p + ht->nNumUsed;
while (1) {
end--;
zv = &end->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
break;
}
if (p == end) goto next;
}
while (p != end) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
}
}
p++;
}
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
}
ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
GC_REF_SET_BLACK(ref);
goto tail_call;
}
next:
ref = GC_STACK_POP(); ref = GC_STACK_POP();
if (ref) { if (ref) {
goto tail_call; goto tail_call;
@ -838,40 +813,30 @@ next:
static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack)
{ {
HashTable *ht = NULL; HashTable *ht;
Bucket *p;
zval *zv; zval *zv;
uint32_t n;
GC_STACK_DCL(stack); GC_STACK_DCL(stack);
do { tail_call:
GC_BENCH_INC(zval_marked_grey); GC_BENCH_INC(zval_marked_grey);
if (GC_TYPE(ref) == IS_OBJECT) { if (GC_TYPE(ref) == IS_OBJECT) {
zend_object *obj = (zend_object*)ref; zend_object *obj = (zend_object*)ref;
if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
int n; zval *table;
zval *zv, *end; int len;
ht = obj->handlers->get_gc(obj, &zv, &n); ht = obj->handlers->get_gc(obj, &table, &len);
n = len;
zv = table;
if (UNEXPECTED(ht)) { if (UNEXPECTED(ht)) {
GC_DELREF(ht); GC_DELREF(ht);
if (!GC_REF_CHECK_COLOR(ht, GC_GREY)) { if (!GC_REF_CHECK_COLOR(ht, GC_GREY)) {
GC_REF_SET_COLOR(ht, GC_GREY); GC_REF_SET_COLOR(ht, GC_GREY);
} else { for (; n != 0; n--) {
ht = NULL;
}
}
if (EXPECTED(!ht)) {
if (!n) goto next;
end = zv + n;
while (!Z_REFCOUNTED_P(--end)) {
if (zv == end) goto next;
}
} else {
if (!n) goto handle_ht;
end = zv + n;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_DELREF(ref); GC_DELREF(ref);
@ -882,76 +847,46 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack)
} }
zv++; zv++;
} }
if (EXPECTED(!ht)) { goto handle_ht;
}
}
handle_zvals:
for (; n != 0; n--) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_DELREF(ref); GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY); GC_REF_SET_COLOR(ref, GC_GREY);
continue; zv++;
while (--n) {
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY);
GC_STACK_PUSH(ref);
} }
goto next;
} }
} else { zv++;
goto next; }
goto tail_call;
}
}
zv++;
}
} }
} else if (GC_TYPE(ref) == IS_ARRAY) { } else if (GC_TYPE(ref) == IS_ARRAY) {
ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table)); ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table));
ht = (zend_array*)ref; ht = (zend_array*)ref;
} else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val);
GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY);
continue;
}
}
goto next;
} else {
goto next;
}
handle_ht: handle_ht:
if (!ht->nNumUsed) goto next; n = ht->nNumUsed;
if (HT_IS_PACKED(ht)) { if (HT_IS_PACKED(ht)) {
zval *end;
zv = ht->arPacked; zv = ht->arPacked;
end = zv + ht->nNumUsed; goto handle_zvals;
while (1) {
end--;
if (Z_REFCOUNTED_P(end)) {
break;
} }
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY);
GC_STACK_PUSH(ref);
}
}
zv++;
}
} else {
Bucket *p = ht->arData;
Bucket *end = p + ht->nNumUsed;
while (1) { p = ht->arData;
end--; for (; n != 0; n--) {
zv = &end->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
break;
}
if (p == end) goto next;
}
while (p != end) {
zv = &p->val; zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) { if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv); zv = Z_INDIRECT_P(zv);
@ -959,6 +894,17 @@ handle_ht:
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_DELREF(ref); GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY);
p++;
while (--n) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY); GC_REF_SET_COLOR(ref, GC_GREY);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
@ -966,21 +912,26 @@ handle_ht:
} }
p++; p++;
} }
zv = &p->val; goto tail_call;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
} }
} }
ref = Z_COUNTED_P(zv); p++;
}
} else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val);
GC_DELREF(ref); GC_DELREF(ref);
if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_GREY); GC_REF_SET_COLOR(ref, GC_GREY);
continue; goto tail_call;
}
}
} }
next:
ref = GC_STACK_POP(); ref = GC_STACK_POP();
} while (ref); if (ref) {
goto tail_call;
}
} }
/* Two-Finger compaction algorithm */ /* Two-Finger compaction algorithm */
@ -1041,7 +992,10 @@ static void gc_mark_roots(gc_stack *stack)
static void gc_scan(zend_refcounted *ref, gc_stack *stack) static void gc_scan(zend_refcounted *ref, gc_stack *stack)
{ {
HashTable *ht;
Bucket *p;
zval *zv; zval *zv;
uint32_t n;
GC_STACK_DCL(stack); GC_STACK_DCL(stack);
tail_call: tail_call:
@ -1066,22 +1020,17 @@ tail_call:
if (GC_TYPE(ref) == IS_OBJECT) { if (GC_TYPE(ref) == IS_OBJECT) {
zend_object *obj = (zend_object*)ref; zend_object *obj = (zend_object*)ref;
if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
int n; zval *table;
zval *zv, *end; int len;
HashTable *ht = obj->handlers->get_gc(obj, &zv, &n);
ht = obj->handlers->get_gc(obj, &table, &len);
n = len;
zv = table;
if (UNEXPECTED(ht)) { if (UNEXPECTED(ht)) {
if (GC_REF_CHECK_COLOR(ht, GC_GREY)) { if (GC_REF_CHECK_COLOR(ht, GC_GREY)) {
GC_REF_SET_COLOR(ht, GC_WHITE); GC_REF_SET_COLOR(ht, GC_WHITE);
GC_STACK_PUSH((zend_refcounted *) ht); GC_STACK_PUSH((zend_refcounted *) ht);
} for (; n != 0; n--) {
}
if (!n) goto next;
end = zv + n;
while (!Z_REFCOUNTED_P(--end)) {
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
@ -1091,60 +1040,62 @@ tail_call:
} }
zv++; zv++;
} }
goto handle_ht;
}
}
handle_zvals:
for (; n != 0; n--) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE); GC_REF_SET_COLOR(ref, GC_WHITE);
zv++;
while (--n) {
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE);
GC_STACK_PUSH(ref);
}
}
zv++;
}
goto tail_call; goto tail_call;
} }
} }
} else if (GC_TYPE(ref) == IS_ARRAY) {
HashTable *ht = (HashTable *)ref;
ZEND_ASSERT(ht != &EG(symbol_table));
if (!ht->nNumUsed) goto next;
if (HT_IS_PACKED(ht)) {
zval *end;
zv = ht->arPacked;
end = zv + ht->nNumUsed;
while (1) {
end--;
if (Z_REFCOUNTED_P(end)) {
break;
}
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE);
GC_STACK_PUSH(ref);
}
}
zv++; zv++;
} }
} else { }
Bucket *p = ht->arData; } else if (GC_TYPE(ref) == IS_ARRAY) {
Bucket *end = p + ht->nNumUsed; ht = (HashTable *)ref;
ZEND_ASSERT(ht != &EG(symbol_table));
while (1) { handle_ht:
end--; n = ht->nNumUsed;
zv = &end->val; if (HT_IS_PACKED(ht)) {
if (Z_TYPE_P(zv) == IS_INDIRECT) { zv = ht->arPacked;
zv = Z_INDIRECT_P(zv); goto handle_zvals;
} }
if (Z_REFCOUNTED_P(zv)) {
break; p = ht->arData;
} for (; n != 0; n--) {
if (p == end) goto next;
}
while (p != end) {
zv = &p->val; zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) { if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv); zv = Z_INDIRECT_P(zv);
} }
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE);
p++;
while (--n) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE); GC_REF_SET_COLOR(ref, GC_WHITE);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
@ -1152,16 +1103,11 @@ tail_call:
} }
p++; p++;
} }
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
}
ref = Z_COUNTED_P(zv);
if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
GC_REF_SET_COLOR(ref, GC_WHITE);
goto tail_call; goto tail_call;
} }
}
p++;
}
} else if (GC_TYPE(ref) == IS_REFERENCE) { } else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) { if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val); ref = Z_COUNTED(((zend_reference*)ref)->val);
@ -1223,11 +1169,13 @@ static void gc_add_garbage(zend_refcounted *ref)
static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *stack) static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *stack)
{ {
int count = 0; int count = 0;
HashTable *ht = NULL; HashTable *ht;
Bucket *p;
zval *zv; zval *zv;
uint32_t n;
GC_STACK_DCL(stack); GC_STACK_DCL(stack);
do { tail_call:
/* don't count references for compatibility ??? */ /* don't count references for compatibility ??? */
if (GC_TYPE(ref) != IS_REFERENCE) { if (GC_TYPE(ref) != IS_REFERENCE) {
count++; count++;
@ -1237,8 +1185,8 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *sta
zend_object *obj = (zend_object*)ref; zend_object *obj = (zend_object*)ref;
if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
int n; int len;
zval *zv, *end; zval *table;
/* optimization: color is GC_BLACK (0) */ /* optimization: color is GC_BLACK (0) */
if (!GC_INFO(ref)) { if (!GC_INFO(ref)) {
@ -1249,26 +1197,14 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *sta
|| obj->ce->destructor != NULL)) { || obj->ce->destructor != NULL)) {
*flags |= GC_HAS_DESTRUCTORS; *flags |= GC_HAS_DESTRUCTORS;
} }
ht = obj->handlers->get_gc(obj, &zv, &n); ht = obj->handlers->get_gc(obj, &table, &len);
n = len;
zv = table;
if (UNEXPECTED(ht)) { if (UNEXPECTED(ht)) {
GC_ADDREF(ht); GC_ADDREF(ht);
if (GC_REF_CHECK_COLOR(ht, GC_WHITE)) { if (GC_REF_CHECK_COLOR(ht, GC_WHITE)) {
GC_REF_SET_BLACK(ht); GC_REF_SET_BLACK(ht);
} else { for (; n != 0; n--) {
ht = NULL;
}
}
if (EXPECTED(!ht)) {
if (!n) goto next;
end = zv + n;
while (!Z_REFCOUNTED_P(--end)) {
if (zv == end) goto next;
}
} else {
if (!n) goto handle_ht;
end = zv + n;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_ADDREF(ref); GC_ADDREF(ref);
@ -1279,17 +1215,34 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *sta
} }
zv++; zv++;
} }
if (EXPECTED(!ht)) { goto handle_ht;
}
}
handle_zvals:
for (; n != 0; n--) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_ADDREF(ref); GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref); GC_REF_SET_BLACK(ref);
continue; zv++;
while (--n) {
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
} }
goto next;
} }
} else { zv++;
goto next; }
goto tail_call;
}
}
zv++;
}
} }
} else if (GC_TYPE(ref) == IS_ARRAY) { } else if (GC_TYPE(ref) == IS_ARRAY) {
/* optimization: color is GC_BLACK (0) */ /* optimization: color is GC_BLACK (0) */
@ -1297,61 +1250,16 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *sta
gc_add_garbage(ref); gc_add_garbage(ref);
} }
ht = (zend_array*)ref; ht = (zend_array*)ref;
} else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val);
GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref);
continue;
}
}
goto next;
} else {
goto next;
}
handle_ht: handle_ht:
if (!ht->nNumUsed) goto next; n = ht->nNumUsed;
if (HT_IS_PACKED(ht)) { if (HT_IS_PACKED(ht)) {
zval *end;
zv = ht->arPacked; zv = ht->arPacked;
end = zv + ht->nNumUsed; goto handle_zvals;
while (1) {
end--;
if (Z_REFCOUNTED_P(end)) {
break;
} }
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref);
}
}
zv++;
}
} else {
Bucket *p = ht->arData;
Bucket *end = p + ht->nNumUsed;
while (1) { p = ht->arData;
end--; for (; n != 0; n--) {
zv = &end->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
break;
}
if (p == end) goto next;
}
while (p != end) {
zv = &p->val; zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) { if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv); zv = Z_INDIRECT_P(zv);
@ -1359,6 +1267,17 @@ handle_ht:
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_ADDREF(ref); GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref);
p++;
while (--n) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref); GC_REF_SET_BLACK(ref);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
@ -1366,21 +1285,26 @@ handle_ht:
} }
p++; p++;
} }
zv = &p->val; goto tail_call;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
} }
} }
ref = Z_COUNTED_P(zv); p++;
}
} else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val);
GC_ADDREF(ref); GC_ADDREF(ref);
if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
GC_REF_SET_BLACK(ref); GC_REF_SET_BLACK(ref);
continue; goto tail_call;
}
}
} }
next:
ref = GC_STACK_POP(); ref = GC_STACK_POP();
} while (ref); if (ref) {
goto tail_call;
}
return count; return count;
} }
@ -1427,12 +1351,14 @@ static int gc_collect_roots(uint32_t *flags, gc_stack *stack)
static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffer *root, gc_stack *stack) static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffer *root, gc_stack *stack)
{ {
HashTable *ht = NULL; HashTable *ht;
Bucket *p;
zval *zv; zval *zv;
uint32_t n;
int count = 0; int count = 0;
GC_STACK_DCL(stack); GC_STACK_DCL(stack);
do { tail_call:
if (root) { if (root) {
root = NULL; root = NULL;
count++; count++;
@ -1444,7 +1370,7 @@ static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffe
} else if (GC_TYPE(ref) == IS_REFERENCE) { } else if (GC_TYPE(ref) == IS_REFERENCE) {
if (Z_REFCOUNTED(((zend_reference*)ref)->val)) { if (Z_REFCOUNTED(((zend_reference*)ref)->val)) {
ref = Z_COUNTED(((zend_reference*)ref)->val); ref = Z_COUNTED(((zend_reference*)ref)->val);
continue; goto tail_call;
} }
goto next; goto next;
} else { } else {
@ -1455,102 +1381,86 @@ static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffe
zend_object *obj = (zend_object*)ref; zend_object *obj = (zend_object*)ref;
if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
int n; int len;
zval *zv, *end; zval *table;
ht = obj->handlers->get_gc(obj, &zv, &n); ht = obj->handlers->get_gc(obj, &table, &len);
if (EXPECTED(!ht)) { n = len;
if (!n) goto next; zv = table;
end = zv + n; if (UNEXPECTED(ht)) {
while (!Z_REFCOUNTED_P(--end)) { for (; n != 0; n--) {
if (zv == end) goto next;
}
} else {
if (!n) goto handle_ht;
end = zv + n;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
} }
zv++; zv++;
} }
if (EXPECTED(!ht)) {
ref = Z_COUNTED_P(zv);
continue;
}
handle_ht:
if (GC_REF_ADDRESS(ht) != 0 && GC_REF_CHECK_COLOR(ht, GC_BLACK)) { if (GC_REF_ADDRESS(ht) != 0 && GC_REF_CHECK_COLOR(ht, GC_BLACK)) {
GC_TRACE_REF(ht, "removing from buffer"); GC_TRACE_REF(ht, "removing from buffer");
GC_REMOVE_FROM_BUFFER(ht); GC_REMOVE_FROM_BUFFER(ht);
} }
} else { goto handle_ht;
goto next;
}
} else if (GC_TYPE(ref) == IS_ARRAY) {
ht = (zend_array*)ref;
} else {
goto next;
} }
if (!ht->nNumUsed) goto next; handle_zvals:
if (HT_IS_PACKED(ht)) { for (; n != 0; n--) {
zval *end;
zv = ht->arPacked;
end = zv + ht->nNumUsed;
while (1) {
end--;
if (Z_REFCOUNTED_P(end)) {
break;
}
if (zv == end) goto next;
}
while (zv != end) {
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
zv++;
while (--n) {
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
} }
zv++; zv++;
} }
} else { goto tail_call;
Bucket *p = ht->arData; }
Bucket *end = p + ht->nNumUsed; zv++;
}
}
} else if (GC_TYPE(ref) == IS_ARRAY) {
ht = (zend_array*)ref;
while (1) { handle_ht:
end--; n = ht->nNumUsed;
zv = &end->val; if (HT_IS_PACKED(ht)) {
if (Z_TYPE_P(zv) == IS_INDIRECT) { zv = ht->arPacked;
zv = Z_INDIRECT_P(zv); goto handle_zvals;
} }
if (Z_REFCOUNTED_P(zv)) {
break; p = ht->arData;
} for (; n != 0; n--) {
if (p == end) goto next;
}
while (p != end) {
zv = &p->val; zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) { if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv); zv = Z_INDIRECT_P(zv);
} }
if (Z_REFCOUNTED_P(zv)) { if (Z_REFCOUNTED_P(zv)) {
ref = Z_COUNTED_P(zv); ref = Z_COUNTED_P(zv);
p++;
while (--n) {
zv = &p->val;
if (Z_TYPE_P(zv) == IS_INDIRECT) {
zv = Z_INDIRECT_P(zv);
}
if (Z_REFCOUNTED_P(zv)) {
zend_refcounted *ref = Z_COUNTED_P(zv);
GC_STACK_PUSH(ref); GC_STACK_PUSH(ref);
} }
p++; p++;
} }
zv = &p->val; goto tail_call;
if (Z_TYPE_P(zv) == IS_INDIRECT) { }
zv = Z_INDIRECT_P(zv); p++;
} }
} }
ref = Z_COUNTED_P(zv);
continue;
next: next:
ref = GC_STACK_POP(); ref = GC_STACK_POP();
} while (ref); if (ref) {
goto tail_call;
}
return count; return count;
} }