mirror of
https://github.com/php/php-src.git
synced 2025-08-16 05:58:45 +02:00
Implement an SSE2 accelerated version of zend_adler32 (#10507)
When benchmarking the file cache of opcache on index.php from a dummy WordPress install, I noticed that 36.42% of the time was spent in zend_adler32 to verify the checksums of the files. Callgrind reported that 332,731,216 instructions were executed during that run and average time to execute the index file was around 91ms. This patch implements an SSE2 accelerated version of zend_adler32, which reduces the number of instructions executed on that bench to 248,600,983, which is a reduction of ~25%. There is also a decrease in wallclock time measurable: around 10ms. Now only 16.05% of the time is spent computing checksums. The benchmark tests were performed using Callgrind, and time for the wallclock time. These tests were executed multiple times and their results were averaged. The WordPress install only contains two almost-blank posts.
This commit is contained in:
parent
d3abcae4a2
commit
722fbd01a3
1 changed files with 66 additions and 14 deletions
|
@ -27,6 +27,11 @@
|
||||||
#include "zend_shared_alloc.h"
|
#include "zend_shared_alloc.h"
|
||||||
#include "zend_observer.h"
|
#include "zend_observer.h"
|
||||||
|
|
||||||
|
#ifdef __SSE2__
|
||||||
|
/* For SSE2 adler32 */
|
||||||
|
#include <immintrin.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
typedef int (*id_function_t)(void *, void *);
|
typedef int (*id_function_t)(void *, void *);
|
||||||
typedef void (*unique_copy_ctor_func_t)(void *pElement);
|
typedef void (*unique_copy_ctor_func_t)(void *pElement);
|
||||||
|
|
||||||
|
@ -451,11 +456,62 @@ zend_op_array* zend_accel_load_script(zend_persistent_script *persistent_script,
|
||||||
#define ADLER32_NMAX 5552
|
#define ADLER32_NMAX 5552
|
||||||
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
|
||||||
|
|
||||||
#define ADLER32_DO1(buf) {s1 += *(buf); s2 += s1;}
|
#define ADLER32_SCALAR_DO1(buf) {s1 += *(buf); s2 += s1;}
|
||||||
#define ADLER32_DO2(buf, i) ADLER32_DO1(buf + i); ADLER32_DO1(buf + i + 1);
|
#define ADLER32_SCALAR_DO2(buf, i) ADLER32_SCALAR_DO1(buf + i); ADLER32_SCALAR_DO1(buf + i + 1);
|
||||||
#define ADLER32_DO4(buf, i) ADLER32_DO2(buf, i); ADLER32_DO2(buf, i + 2);
|
#define ADLER32_SCALAR_DO4(buf, i) ADLER32_SCALAR_DO2(buf, i); ADLER32_SCALAR_DO2(buf, i + 2);
|
||||||
#define ADLER32_DO8(buf, i) ADLER32_DO4(buf, i); ADLER32_DO4(buf, i + 4);
|
#define ADLER32_SCALAR_DO8(buf, i) ADLER32_SCALAR_DO4(buf, i); ADLER32_SCALAR_DO4(buf, i + 4);
|
||||||
#define ADLER32_DO16(buf) ADLER32_DO8(buf, 0); ADLER32_DO8(buf, 8);
|
#define ADLER32_SCALAR_DO16(buf) ADLER32_SCALAR_DO8(buf, 0); ADLER32_SCALAR_DO8(buf, 8);
|
||||||
|
|
||||||
|
static zend_always_inline void adler32_do16_loop(unsigned char *buf, unsigned char *end, unsigned int *s1_out, unsigned int *s2_out)
|
||||||
|
{
|
||||||
|
unsigned int s1 = *s1_out;
|
||||||
|
unsigned int s2 = *s2_out;
|
||||||
|
|
||||||
|
#ifdef __SSE2__
|
||||||
|
const __m128i zero = _mm_setzero_si128();
|
||||||
|
|
||||||
|
__m128i accumulate_s2 = zero;
|
||||||
|
unsigned int accumulate_s1 = 0;
|
||||||
|
|
||||||
|
do {
|
||||||
|
__m128i read = _mm_loadu_si128((__m128i *) buf); /* [A:P] */
|
||||||
|
|
||||||
|
/* Split the 8-bit-element vector into two 16-bit-element vectors where each element gets zero-extended from 8-bits to 16-bits */
|
||||||
|
__m128i lower = _mm_unpacklo_epi8(read, zero); /* [A:H] zero-extended to 16-bits */
|
||||||
|
__m128i higher = _mm_unpackhi_epi8(read, zero); /* [I:P] zero-extended to 16-bits */
|
||||||
|
lower = _mm_madd_epi16(lower, _mm_set_epi16(9, 10, 11, 12, 13, 14, 15, 16)); /* [A * 16:H * 9] */
|
||||||
|
higher = _mm_madd_epi16(higher, _mm_set_epi16(1, 2, 3, 4, 5, 6, 7, 8)); /* [I * 8:P * 1] */
|
||||||
|
|
||||||
|
/* We'll cheat here: it's difficult to add 16-bit elementwise, but we can do 32-bit additions.
|
||||||
|
* The highest value the sum of two elements of the vectors can take is 0xff * 16 + 0xff * 8 < 0xffff.
|
||||||
|
* That means there is no carry possible from 16->17 bits so the 32-bit addition is safe. */
|
||||||
|
__m128i sum = _mm_add_epi32(lower, higher); /* [A * 16 + I * 8:H * 9 + P * 1] */
|
||||||
|
accumulate_s2 = _mm_add_epi32(accumulate_s2, sum);
|
||||||
|
accumulate_s1 += s1;
|
||||||
|
|
||||||
|
/* Computes 8-bit element-wise abs(buf - zero) and then sums the elements into two 16 bit parts */
|
||||||
|
sum = _mm_sad_epu8(read, zero);
|
||||||
|
s1 += _mm_cvtsi128_si32(sum) + _mm_extract_epi16(sum, 4);
|
||||||
|
|
||||||
|
buf += 16;
|
||||||
|
} while (buf != end);
|
||||||
|
|
||||||
|
/* For convenience, let's do a rename of variables and let accumulate_s2 = [X, Y, Z, W] */
|
||||||
|
__m128i shuffled = _mm_shuffle_epi32(accumulate_s2, _MM_SHUFFLE(1, 0, 0, 2)); /* [Y, X, X, Z] */
|
||||||
|
accumulate_s2 = _mm_add_epi32(accumulate_s2, shuffled); /* [X + Y, Y + X, Z + X, W + Z] */
|
||||||
|
shuffled = _mm_shuffle_epi32(accumulate_s2, _MM_SHUFFLE(3, 3, 3, 3)); /* [X + Y, X + Y, X + Y, X + Y] */
|
||||||
|
accumulate_s2 = _mm_add_epi32(accumulate_s2, shuffled); /* [/, /, /, W + Z + X + Y] */
|
||||||
|
s2 += accumulate_s1 * 16 + _mm_cvtsi128_si32(accumulate_s2);
|
||||||
|
#else
|
||||||
|
do {
|
||||||
|
ADLER32_SCALAR_DO16(buf);
|
||||||
|
buf += 16;
|
||||||
|
} while (buf != end);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
*s1_out = s1;
|
||||||
|
*s2_out = s2;
|
||||||
|
}
|
||||||
|
|
||||||
unsigned int zend_adler32(unsigned int checksum, unsigned char *buf, uint32_t len)
|
unsigned int zend_adler32(unsigned int checksum, unsigned char *buf, uint32_t len)
|
||||||
{
|
{
|
||||||
|
@ -466,10 +522,8 @@ unsigned int zend_adler32(unsigned int checksum, unsigned char *buf, uint32_t le
|
||||||
while (len >= ADLER32_NMAX) {
|
while (len >= ADLER32_NMAX) {
|
||||||
len -= ADLER32_NMAX;
|
len -= ADLER32_NMAX;
|
||||||
end = buf + ADLER32_NMAX;
|
end = buf + ADLER32_NMAX;
|
||||||
do {
|
adler32_do16_loop(buf, end, &s1, &s2);
|
||||||
ADLER32_DO16(buf);
|
buf = end;
|
||||||
buf += 16;
|
|
||||||
} while (buf != end);
|
|
||||||
s1 %= ADLER32_BASE;
|
s1 %= ADLER32_BASE;
|
||||||
s2 %= ADLER32_BASE;
|
s2 %= ADLER32_BASE;
|
||||||
}
|
}
|
||||||
|
@ -478,15 +532,13 @@ unsigned int zend_adler32(unsigned int checksum, unsigned char *buf, uint32_t le
|
||||||
if (len >= 16) {
|
if (len >= 16) {
|
||||||
end = buf + (len & 0xfff0);
|
end = buf + (len & 0xfff0);
|
||||||
len &= 0xf;
|
len &= 0xf;
|
||||||
do {
|
adler32_do16_loop(buf, end, &s1, &s2);
|
||||||
ADLER32_DO16(buf);
|
buf = end;
|
||||||
buf += 16;
|
|
||||||
} while (buf != end);
|
|
||||||
}
|
}
|
||||||
if (len) {
|
if (len) {
|
||||||
end = buf + len;
|
end = buf + len;
|
||||||
do {
|
do {
|
||||||
ADLER32_DO1(buf);
|
ADLER32_SCALAR_DO1(buf);
|
||||||
buf++;
|
buf++;
|
||||||
} while (buf != end);
|
} while (buf != end);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue