Make distinct between indexes/numbers, GC addresses and pointers to gc_root_buffers.

Perform conversion through macros.

Zend/zend_gc.c

@@ -119,24 +119,6 @@
 		GC_TYPE_INFO(ref) |= (GC_COLOR << GC_INFO_SHIFT); \
 	} while (0)
 
-/* GC buffer size */
-#define GC_INVALID           0
-#define GC_FIRST_REAL_ROOT   1
-
-#define GC_DEFAULT_BUF_SIZE  (16 * 1024)
-#define GC_BUF_GROW_STEP     (128 * 1024)
-
-#define GC_MAX_UNCOMPRESSED  (1024 * 1024)
-#define GC_MAX_BUF_SIZE      0x40000000
-
-#define GC_THRESHOLD_DEFAULT 10000
-#define GC_THRESHOLD_STEP    10000
-#define GC_THRESHOLD_MAX     1000000000
-#define GC_THRESHOLD_TRIGGER 100
-
-/* GC flags */
-#define GC_HAS_DESTRUCTORS  (1<<0)
-
 /* bit stealing tags for gc_root_buffer.ref */
 #define GC_BITS    0x3
 
@@ -157,9 +139,43 @@
 #define GC_MAKE_GARBAGE(ptr) \
 	((void*)(((uintptr_t)(ptr)) | GC_GARBAGE))
 
+/* GC address conversion */
+#define GC_ADDR2NUM(addr)    (addr)
+#define GC_NUM2ADDR(num)     (num)
+
+#define GC_NEXT_ADDR(addr)   ((addr) + 1)
+#define GC_PREV_ADDR(addr)   ((addr) - 1)
+
+#define GC_ADDR2PTR(addr)    (GC_G(buf) + (addr))
+#define GC_PTR2ADDR(ptr)     ((ptr) - GC_G(buf))
+
+#define GC_ADDR2LIST(addr)   ((void*)(uintptr_t)(((addr) * sizeof(void*)) | GC_UNUSED))
+#define GC_LIST2ADDR(list)   (((uint32_t)(uintptr_t)(list)) / sizeof(void*))
+
+/* GC buffers */
+#define GC_INVALID_NUM       0
+#define GC_FIRST_ROOT_NUM    1
+
+#define GC_INVALID_ADDR      GC_NUM2ADDR(GC_INVALID_NUM)
+#define GC_FIRST_ROOT_ADDR   GC_NUM2ADDR(GC_FIRST_ROOT_NUM)
+
+#define GC_DEFAULT_BUF_SIZE  (16 * 1024)
+#define GC_BUF_GROW_STEP     (128 * 1024)
+
+#define GC_MAX_UNCOMPRESSED  (1024 * 1024)
+#define GC_MAX_BUF_SIZE      0x40000000
+
+#define GC_THRESHOLD_DEFAULT 10000
+#define GC_THRESHOLD_STEP    10000
+#define GC_THRESHOLD_MAX     1000000000
+#define GC_THRESHOLD_TRIGGER 100
+
+/* GC flags */
+#define GC_HAS_DESTRUCTORS  (1<<0)
+
 /* unused buffers */
 #define GC_HAS_UNUSED() \
-	(GC_G(unused) != GC_INVALID)
+	(GC_G(unused) != GC_INVALID_ADDR)
 #define GC_FETCH_UNUSED() \
 	gc_fetch_unused()
 #define GC_LINK_UNUSED(root) \
@@ -185,10 +201,10 @@ typedef struct _zend_gc_globals {
 	zend_bool         gc_full;
 
 	gc_root_buffer   *buf;				/* preallocated arrays of buffers   */
-	uint32_t          buf_size;			/* size of the GC buffer            */
-	uint32_t          gc_threshold;     /* GC collection threshold          */
 	uint32_t          unused;			/* linked list of unused buffers    */
 	uint32_t          first_unused;		/* first unused buffer              */
+	uint32_t          gc_threshold;     /* GC collection threshold [addr]   */
+	uint32_t          buf_size;			/* size of the GC buffer [addr]     */
 	uint32_t          num_roots;		/* number of roots in GC buffer     */
 
 	uint32_t gc_runs;
@@ -248,7 +264,7 @@ static zend_always_inline uint32_t gc_compress(uint32_t addr)
 
 static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t addr)
 {
-	gc_root_buffer *root = GC_G(buf) + addr;
+	gc_root_buffer *root = GC_ADDR2PTR(addr);
 
 	if (EXPECTED(GC_GET_PTR(root->ref) == ref)) {
 		return root;
@@ -257,7 +273,7 @@ static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, ui
 	while (1) {
 		addr += GC_MAX_UNCOMPRESSED;
 		ZEND_ASSERT(addr < GC_G(first_unused));
-		root = GC_G(buf) + addr;
+		root = GC_ADDR2PTR(addr);
 		if (GC_GET_PTR(root->ref) == ref) {
 			return root;
 		}
@@ -271,17 +287,16 @@ static zend_always_inline uint32_t gc_fetch_unused(void)
 
 	ZEND_ASSERT(GC_HAS_UNUSED());
 	addr = GC_G(unused);
-	root = GC_G(buf) + addr;
+	root = GC_ADDR2PTR(addr);
 	ZEND_ASSERT(GC_IS_UNUSED(root->ref));
-	/* optimization: GC_GET_PTR(root->ref) is not necessary because it shifted anyway */
+	GC_G(unused) = GC_LIST2ADDR(root->ref);
-	GC_G(unused) = (uint32_t)(uintptr_t)root->ref / sizeof(void*);
 	return addr;
 }
 
 static zend_always_inline void gc_link_unused(gc_root_buffer *root)
 {
-	root->ref = (void*)(uintptr_t)((GC_G(unused) * sizeof(void*)) | GC_UNUSED);
+	root->ref = GC_ADDR2LIST(GC_G(unused));
-	GC_G(unused) = root - GC_G(buf);
+	GC_G(unused) = GC_PTR2ADDR(root);
 }
 
 static zend_always_inline uint32_t gc_fetch_next_unused(void)
@@ -290,7 +305,7 @@ static zend_always_inline uint32_t gc_fetch_next_unused(void)
 
 	ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
 	addr = GC_G(first_unused);
-	GC_G(first_unused)++;
+	GC_G(first_unused) = GC_NEXT_ADDR(GC_G(first_unused));
 	return addr;
 }
 
@@ -349,10 +364,10 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals)
 	gc_globals->gc_full = 0;
 
 	gc_globals->buf = NULL;
-	gc_globals->buf_size = 0;
+	gc_globals->unused = GC_INVALID_ADDR;
-	gc_globals->gc_threshold = 0;
+	gc_globals->first_unused = GC_INVALID_ADDR;
-	gc_globals->unused = GC_INVALID;
+	gc_globals->gc_threshold = GC_INVALID_ADDR;
-	gc_globals->first_unused = 0;
+	gc_globals->buf_size = GC_INVALID_ADDR;
 	gc_globals->num_roots = 0;
 
 	gc_globals->gc_runs = 0;
@@ -390,8 +405,8 @@ void gc_reset(void)
 		GC_G(gc_active) = 0;
 		GC_G(gc_protected) = 0;
 		GC_G(gc_full) = 0;
-		GC_G(unused) = GC_INVALID;
+		GC_G(unused) = GC_INVALID_ADDR;
-		GC_G(first_unused) = GC_FIRST_REAL_ROOT;
+		GC_G(first_unused) = GC_FIRST_ROOT_ADDR;
 		GC_G(num_roots) = 0;
 
 		GC_G(gc_runs) = 0;
@@ -414,8 +429,8 @@ ZEND_API zend_bool gc_enable(zend_bool enable)
 	GC_G(gc_enabled) = enable;
 	if (enable && !old_enabled && GC_G(buf) == NULL) {
 		GC_G(buf) = (gc_root_buffer*) malloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE);
-		GC_G(buf_size) = GC_DEFAULT_BUF_SIZE;
+		GC_G(buf_size) = GC_NUM2ADDR(GC_DEFAULT_BUF_SIZE);
-		GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT + GC_FIRST_REAL_ROOT;
+		GC_G(gc_threshold) = GC_NUM2ADDR(GC_THRESHOLD_DEFAULT + GC_FIRST_ROOT_NUM);
 		gc_reset();
 	}
 	return old_enabled;
@@ -442,7 +457,7 @@ static void gc_grow_root_buffer(void)
 {
 	size_t new_size;
 
-	if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) {
+	if (GC_ADDR2NUM(GC_G(buf_size)) >= GC_MAX_BUF_SIZE) {
 		if (!GC_G(gc_full)) {
 			zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n");
 			GC_G(gc_active) = 1;
@@ -451,16 +466,16 @@ static void gc_grow_root_buffer(void)
 			return;
 		}
 	}
-	if (GC_G(buf_size) < GC_BUF_GROW_STEP) {
+	if (GC_G(buf_size) < GC_NUM2ADDR(GC_BUF_GROW_STEP)) {
-		new_size = GC_G(buf_size) * 2;
+		new_size = GC_ADDR2NUM(GC_G(buf_size)) * 2;
 	} else {
-		new_size = GC_G(buf_size) + GC_BUF_GROW_STEP;
+		new_size = GC_ADDR2NUM(GC_G(buf_size)) + GC_BUF_GROW_STEP;
 	}
 	if (new_size > GC_MAX_BUF_SIZE) {
 		new_size = GC_MAX_BUF_SIZE;
 	}
 	GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1);
-	GC_G(buf_size) = new_size;
+	GC_G(buf_size) = GC_NUM2ADDR(new_size);
 }
 
 static void gc_adjust_threshold(int count)
@@ -472,24 +487,24 @@ static void gc_adjust_threshold(int count)
 	 * by a fixed step */
 	if (count < GC_THRESHOLD_TRIGGER) {
 		/* increase */
-		if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) {
+		if (GC_G(gc_threshold) < GC_NUM2ADDR(GC_THRESHOLD_MAX)) {
-			new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP;
+			new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) + GC_THRESHOLD_STEP;
 			if (new_threshold > GC_THRESHOLD_MAX) {
 				new_threshold = GC_THRESHOLD_MAX;
 			}
-			if (new_threshold > GC_G(buf_size)) {
+			if (new_threshold > GC_ADDR2NUM(GC_G(buf_size))) {
 				gc_grow_root_buffer();
 			}
-			if (new_threshold <= GC_G(buf_size)) {
+			if (new_threshold <= GC_ADDR2NUM(GC_G(buf_size))) {
-				GC_G(gc_threshold) = new_threshold;
+				GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold);
 			}
 		}
-	} else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) {
+	} else if (GC_G(gc_threshold) > GC_NUM2ADDR(GC_THRESHOLD_DEFAULT)) {
-		new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP;
+		new_threshold = GC_ADDR2NUM(GC_G(gc_threshold)) - GC_THRESHOLD_STEP;
 		if (new_threshold < GC_THRESHOLD_DEFAULT) {
 			new_threshold = GC_THRESHOLD_DEFAULT;
 		}
-		GC_G(gc_threshold) = new_threshold;
+		GC_G(gc_threshold) = GC_NUM2ADDR(new_threshold);
 	}
 }
 
@@ -518,11 +533,13 @@ static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refc
 		addr = GC_FETCH_NEXT_UNUSED();
 	} else {
 		gc_grow_root_buffer();
-		ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
+		if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
+			return;
+		}
 		addr = GC_FETCH_NEXT_UNUSED();
 	}
 
-	newRoot = GC_G(buf) + addr;
+	newRoot = GC_ADDR2PTR(addr);
 	newRoot->ref = ref; /* GC_ROOT tag is 0 */
 	GC_TRACE_SET_COLOR(ref, GC_PURPLE);
 
@@ -558,7 +575,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref)
 	ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
 	ZEND_ASSERT(GC_INFO(ref) == 0);
 
-	newRoot = GC_G(buf) + addr;
+	newRoot = GC_ADDR2PTR(addr);
 	newRoot->ref = ref; /* GC_ROOT tag is 0 */
 	GC_TRACE_SET_COLOR(ref, GC_PURPLE);
 
@@ -592,12 +609,12 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
 	GC_REF_SET_INFO(ref, 0);
 
 	/* Perform decopression only in case of large buffers */
-	if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) {
+	if (UNEXPECTED(GC_G(first_unused) >= GC_NUM2ADDR(GC_MAX_UNCOMPRESSED))) {
 		gc_remove_compressed(ref, addr);
 		return;
 	}
 
-	root = GC_G(buf) + addr;
+	root = GC_ADDR2PTR(addr);
 	gc_remove_from_roots(root);
 }
 
@@ -813,30 +830,30 @@ tail_call:
 /* Two-Finger compaction algorithm */
 static void gc_compact(void)
 {
-	if (GC_G(num_roots) + GC_FIRST_REAL_ROOT != GC_G(first_unused)) {
+	if (GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM) != GC_G(first_unused)) {
 		if (GC_G(num_roots)) {
-			gc_root_buffer *buf = GC_G(buf);
+			gc_root_buffer *free = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-			uint32_t free = GC_FIRST_REAL_ROOT;
+			gc_root_buffer *scan = GC_ADDR2PTR(GC_PREV_ADDR(GC_G(first_unused)));
-			uint32_t scan = GC_G(first_unused) - 1;
+			gc_root_buffer *end  = GC_ADDR2PTR(GC_NUM2ADDR(GC_G(num_roots)));
 			uint32_t addr;
 			zend_refcounted *p;
 
 			while (free < scan) {
-				while (!GC_IS_UNUSED(buf[free].ref)) {
+				while (!GC_IS_UNUSED(free->ref)) {
 					free++;
 				}
-				while (GC_IS_UNUSED(buf[scan].ref)) {
+				while (GC_IS_UNUSED(scan->ref)) {
 					scan--;
 				}
 				if (scan > free) {
-					p = buf[scan].ref;
+					p = scan->ref;
-					buf[free].ref = p;
+					free->ref = p;
 					p = GC_GET_PTR(p);
-					addr = gc_compress(free);
+					addr = gc_compress(GC_PTR2ADDR(free));
 					GC_REF_SET_INFO(p, addr | GC_REF_COLOR(p));
 					free++;
 					scan--;
-					if (scan <= GC_G(num_roots)) {
+					if (scan <= end) {
 						break;
 					}
 				} else {
@@ -844,8 +861,8 @@ static void gc_compact(void)
 				}
 			}
 		}
-		GC_G(unused) = GC_INVALID;
+		GC_G(unused) = GC_INVALID_ADDR;
-		GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_REAL_ROOT;
+		GC_G(first_unused) = GC_NUM2ADDR(GC_G(num_roots) + GC_FIRST_ROOT_NUM);
 	}
 }
 
@@ -855,8 +872,8 @@ static void gc_mark_roots(void)
 
 	gc_compact();
 
-	current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+	current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);;
-	last = GC_G(buf) + GC_G(first_unused);
+	last = GC_ADDR2PTR(GC_G(first_unused));
 	while (current != last) {
 		if (GC_IS_ROOT(current->ref)) {
 			if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) {
@@ -966,8 +983,8 @@ tail_call:
 
 static void gc_scan_roots(void)
 {
-	gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+	gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-	gc_root_buffer *last = GC_G(buf) + GC_G(first_unused);
+	gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused));
 
 	while (current != last) {
 		if (GC_IS_ROOT(current->ref)) {
@@ -988,11 +1005,13 @@ static void gc_add_garbage(zend_refcounted *ref)
 		addr = GC_FETCH_NEXT_UNUSED();
 	} else {
 		gc_grow_root_buffer();
-		ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
+		if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
+			return;
+		}
 		addr = GC_FETCH_NEXT_UNUSED();
 	}
 
-	buf = GC_G(buf) + addr;
+	buf = GC_ADDR2PTR(addr);
 	buf->ref = GC_MAKE_GARBAGE(ref);
 
 	addr = gc_compress(addr);
@@ -1131,11 +1150,11 @@ tail_call:
 
 static int gc_collect_roots(uint32_t *flags)
 {
-	uint32_t n, end;
+	uint32_t addr, end;
 	zend_refcounted *ref;
 	int count = 0;
-	gc_root_buffer *current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+	gc_root_buffer *current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-	gc_root_buffer *last = GC_G(buf) + GC_G(first_unused);
+	gc_root_buffer *last = GC_ADDR2PTR(GC_G(first_unused));
 
 	/* remove non-garbage from the list */
 	while (current != last) {
@@ -1152,10 +1171,10 @@ static int gc_collect_roots(uint32_t *flags)
 
 	/* Root buffer might be reallocated during gc_collect_white,
 	 * make sure to reload pointers. */
-	n = GC_FIRST_REAL_ROOT;
+	addr = GC_FIRST_ROOT_ADDR;
 	end = GC_G(first_unused);
-	while (n != end) {
+	while (addr != end) {
-		current = GC_G(buf) + n;
+		current = GC_ADDR2PTR(addr);
 		ref = current->ref;
 		if (GC_IS_ROOT(ref)) {
 			if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
@@ -1163,7 +1182,7 @@ static int gc_collect_roots(uint32_t *flags)
 				count += gc_collect_white(ref, flags);
 			}
 		}
-		n++;
+		addr = GC_NEXT_ADDR(addr);
 	}
 
 	return count;
@@ -1275,7 +1294,7 @@ ZEND_API int zend_gc_collect_cycles(void)
 		gc_root_buffer *current, *last;
 		zend_refcounted *p;
 		uint32_t gc_flags = 0;
-		uint32_t n, end;
+		uint32_t addr, end;
 
 		if (GC_G(gc_active)) {
 			return 0;
@@ -1303,17 +1322,18 @@ ZEND_API int zend_gc_collect_cycles(void)
 		end = GC_G(first_unused);
 
 		if (gc_flags & GC_HAS_DESTRUCTORS) {
-			uint32_t *refcounts;
+			uint32_t *refcounts, count, n;
 
 			GC_TRACE("Calling destructors");
 
 			// TODO: may be use emalloc() ???
-			refcounts = pemalloc(sizeof(uint32_t) * GC_G(first_unused), 1);
+			count = GC_ADDR2NUM(GC_G(first_unused));
+			refcounts = pemalloc(sizeof(uint32_t) * count, 1);
 
 			/* Remember reference counters before calling destructors */
-			n = GC_FIRST_REAL_ROOT;
+			n = GC_FIRST_ROOT_NUM;
-			current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+			current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-			while (n != end) {
+			while (n != count) {
 				if (GC_IS_GARBAGE(current->ref)) {
 					p = GC_GET_PTR(current->ref);
 					refcounts[n] = GC_REFCOUNT(p);
@@ -1326,9 +1346,9 @@ ZEND_API int zend_gc_collect_cycles(void)
 			 *
 			 * The root buffer might be reallocated during destructors calls,
 			 * make sure to reload pointers as necessary. */
-			n = GC_FIRST_REAL_ROOT;
+			addr = GC_FIRST_ROOT_ADDR;
-			while (n != end) {
+			while (addr != end) {
-				current = GC_G(buf) + n;
+				current = GC_ADDR2PTR(addr);
 				if (GC_IS_GARBAGE(current->ref)) {
 					p = GC_GET_PTR(current->ref);
 					if (GC_TYPE(p) == IS_OBJECT
@@ -1346,13 +1366,13 @@ ZEND_API int zend_gc_collect_cycles(void)
 						}
 					}
 				}
-				n++;
+				addr = GC_NEXT_ADDR(addr);
 			}
 
 			/* Remove values captured in destructors */
-			n = GC_FIRST_REAL_ROOT;
+			n = GC_FIRST_ROOT_NUM;
-			current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+			current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-			while (n != end) {
+			while (n != count) {
 				if (GC_IS_GARBAGE(current->ref)) {
 					p = GC_GET_PTR(current->ref);
 					if (GC_REFCOUNT(p) > refcounts[n]) {
@@ -1374,8 +1394,8 @@ ZEND_API int zend_gc_collect_cycles(void)
 		/* Destroy zvals */
 		GC_TRACE("Destroying zvals");
 		GC_G(gc_protected) = 1;
-		current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+		current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
-		last = GC_G(buf) + GC_G(first_unused);
+		last = GC_ADDR2PTR(GC_G(first_unused));
 		while (current != last) {
 			if (GC_IS_GARBAGE(current->ref)) {
 				p = GC_GET_PTR(current->ref);
@@ -1413,7 +1433,7 @@ ZEND_API int zend_gc_collect_cycles(void)
 		}
 
 		/* Free objects */
-		current = GC_G(buf) + GC_FIRST_REAL_ROOT;
+		current = GC_ADDR2PTR(GC_FIRST_ROOT_ADDR);
 		while (current != last) {
 			if (GC_IS_GARBAGE(current->ref)) {
 				p = GC_GET_PTR(current->ref);