From c79ce48ddb2125a3d19723e0fbca1bd770bacab7 Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 3 Mar 2019 19:57:40 -0800 Subject: [PATCH] Fix #71890: Add support for crc32c Castagnoli's polynomial. This variant of crc32 is heavily used by storage systems, such as iSCSI, SCTP, Btrfs, ext4, and is increasingly being used in API (such as Google Cloud Storage, and Apache Kafka). --- NEWS | 1 + UPGRADING | 4 ++ ext/hash/hash.c | 8 +-- ext/hash/hash_crc32.c | 28 ++++++++-- ext/hash/php_hash.h | 1 + ext/hash/php_hash_crc32.h | 4 +- ext/hash/php_hash_crc32_tables.h | 68 +++++++++++++++++++++++++ ext/hash/tests/crc32.phpt | 85 +++++++++++++++++++++++++++++++ ext/hash/tests/hash-clone.phpt | 6 +++ ext/hash/tests/hash_algos.phpt | 4 +- ext/hash/tests/hash_copy_001.phpt | 6 +++ 11 files changed, 206 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 9bf6fdb3c79..1e2103ef96a 100644 --- a/NEWS +++ b/NEWS @@ -43,6 +43,7 @@ PHP NEWS - Hash: . The hash extension is now an integral part of PHP and cannot be disabled as per RFC: https://wiki.php.net/rfc/permanent_hash_ext. (Kalle) + . Implemented FR #71890 (crc32c checksum algorithm). (Andrew Brampton) - Intl: . Raised requirements to ICU ≥ 50.1. (cmb) diff --git a/UPGRADING b/UPGRADING index e20a9e2d42e..ded81d175bc 100644 --- a/UPGRADING +++ b/UPGRADING @@ -117,6 +117,10 @@ PHP 7.4 UPGRADE NOTES native variables and create/access data structures defined in C libraries. RFC: https://wiki.php.net/rfc/ffi +- Hash: + . Added "crc32c" hash using Castagnoli's polynomial. This crc32 variant is + used by storage systems, such as iSCSI, SCTP, Btrfs and ext4. + - Mbstring: . Added mb_str_split() function, which provide the same functionality as str_split(), but operating on code points rather than bytes. diff --git a/ext/hash/hash.c b/ext/hash/hash.c index 1c1d0b4ece4..449665a2685 100644 --- a/ext/hash/hash.c +++ b/ext/hash/hash.c @@ -40,10 +40,10 @@ struct mhash_bc_entry { int value; }; -#define MHASH_NUM_ALGOS 34 +#define MHASH_NUM_ALGOS 35 static struct mhash_bc_entry mhash_to_hash[MHASH_NUM_ALGOS] = { - {"CRC32", "crc32", 0}, + {"CRC32", "crc32", 0}, /* used by bzip */ {"MD5", "md5", 1}, {"SHA1", "sha1", 2}, {"HAVAL256", "haval256,3", 3}, @@ -52,7 +52,7 @@ static struct mhash_bc_entry mhash_to_hash[MHASH_NUM_ALGOS] = { {NULL, NULL, 6}, {"TIGER", "tiger192,3", 7}, {"GOST", "gost", 8}, - {"CRC32B", "crc32b", 9}, + {"CRC32B", "crc32b", 9}, /* used by ethernet (IEEE 802.3), gzip, zip, png, etc */ {"HAVAL224", "haval224,3", 10}, {"HAVAL192", "haval192,3", 11}, {"HAVAL160", "haval160,3", 12}, @@ -77,6 +77,7 @@ static struct mhash_bc_entry mhash_to_hash[MHASH_NUM_ALGOS] = { {"FNV164", "fnv164", 31}, {"FNV1A64", "fnv1a64", 32}, {"JOAAT", "joaat", 33}, + {"CRC32C", "crc32c", 34}, /* Castagnoli's CRC, used by iSCSI, SCTP, Btrfs, ext4, etc */ }; #endif @@ -1204,6 +1205,7 @@ PHP_MINIT_FUNCTION(hash) php_hash_register_algo("adler32", &php_hash_adler32_ops); php_hash_register_algo("crc32", &php_hash_crc32_ops); php_hash_register_algo("crc32b", &php_hash_crc32b_ops); + php_hash_register_algo("crc32c", &php_hash_crc32c_ops); php_hash_register_algo("fnv132", &php_hash_fnv132_ops); php_hash_register_algo("fnv1a32", &php_hash_fnv1a32_ops); php_hash_register_algo("fnv164", &php_hash_fnv164_ops); diff --git a/ext/hash/hash_crc32.c b/ext/hash/hash_crc32.c index 4f095e4b715..95f3600dca4 100644 --- a/ext/hash/hash_crc32.c +++ b/ext/hash/hash_crc32.c @@ -44,7 +44,16 @@ PHP_HASH_API void PHP_CRC32BUpdate(PHP_CRC32_CTX *context, const unsigned char * } } -PHP_HASH_API void PHP_CRC32Final(unsigned char digest[4], PHP_CRC32_CTX *context) +PHP_HASH_API void PHP_CRC32CUpdate(PHP_CRC32_CTX *context, const unsigned char *input, size_t len) +{ + size_t i; + + for (i = 0; i < len; ++i) { + context->state = (context->state >> 8) ^ crc32c_table[(context->state ^ input[i]) & 0xff]; + } +} + +PHP_HASH_API void PHP_CRC32LEFinal(unsigned char digest[4], PHP_CRC32_CTX *context) { context->state=~context->state; digest[3] = (unsigned char) ((context->state >> 24) & 0xff); @@ -54,7 +63,7 @@ PHP_HASH_API void PHP_CRC32Final(unsigned char digest[4], PHP_CRC32_CTX *context context->state = 0; } -PHP_HASH_API void PHP_CRC32BFinal(unsigned char digest[4], PHP_CRC32_CTX *context) +PHP_HASH_API void PHP_CRC32BEFinal(unsigned char digest[4], PHP_CRC32_CTX *context) { context->state=~context->state; digest[0] = (unsigned char) ((context->state >> 24) & 0xff); @@ -73,7 +82,7 @@ PHP_HASH_API int PHP_CRC32Copy(const php_hash_ops *ops, PHP_CRC32_CTX *orig_cont const php_hash_ops php_hash_crc32_ops = { (php_hash_init_func_t) PHP_CRC32Init, (php_hash_update_func_t) PHP_CRC32Update, - (php_hash_final_func_t) PHP_CRC32Final, + (php_hash_final_func_t) PHP_CRC32LEFinal, (php_hash_copy_func_t) PHP_CRC32Copy, 4, /* what to say here? */ 4, @@ -84,7 +93,18 @@ const php_hash_ops php_hash_crc32_ops = { const php_hash_ops php_hash_crc32b_ops = { (php_hash_init_func_t) PHP_CRC32Init, (php_hash_update_func_t) PHP_CRC32BUpdate, - (php_hash_final_func_t) PHP_CRC32BFinal, + (php_hash_final_func_t) PHP_CRC32BEFinal, + (php_hash_copy_func_t) PHP_CRC32Copy, + 4, /* what to say here? */ + 4, + sizeof(PHP_CRC32_CTX), + 0 +}; + +const php_hash_ops php_hash_crc32c_ops = { + (php_hash_init_func_t) PHP_CRC32Init, + (php_hash_update_func_t) PHP_CRC32CUpdate, + (php_hash_final_func_t) PHP_CRC32BEFinal, (php_hash_copy_func_t) PHP_CRC32Copy, 4, /* what to say here? */ 4, diff --git a/ext/hash/php_hash.h b/ext/hash/php_hash.h index 8f23c1920d8..8e919e2a6fa 100644 --- a/ext/hash/php_hash.h +++ b/ext/hash/php_hash.h @@ -91,6 +91,7 @@ extern const php_hash_ops php_hash_gost_crypto_ops; extern const php_hash_ops php_hash_adler32_ops; extern const php_hash_ops php_hash_crc32_ops; extern const php_hash_ops php_hash_crc32b_ops; +extern const php_hash_ops php_hash_crc32c_ops; extern const php_hash_ops php_hash_fnv132_ops; extern const php_hash_ops php_hash_fnv1a32_ops; extern const php_hash_ops php_hash_fnv164_ops; diff --git a/ext/hash/php_hash_crc32.h b/ext/hash/php_hash_crc32.h index 5b187dc51f3..6308b005e32 100644 --- a/ext/hash/php_hash_crc32.h +++ b/ext/hash/php_hash_crc32.h @@ -28,7 +28,9 @@ typedef struct { PHP_HASH_API void PHP_CRC32Init(PHP_CRC32_CTX *context); PHP_HASH_API void PHP_CRC32Update(PHP_CRC32_CTX *context, const unsigned char *input, size_t len); PHP_HASH_API void PHP_CRC32BUpdate(PHP_CRC32_CTX *context, const unsigned char *input, size_t len); -PHP_HASH_API void PHP_CRC32Final(unsigned char digest[4], PHP_CRC32_CTX *context); +PHP_HASH_API void PHP_CRC32CUpdate(PHP_CRC32_CTX *context, const unsigned char *input, size_t len); +PHP_HASH_API void PHP_CRC32LEFinal(unsigned char digest[4], PHP_CRC32_CTX *context); +PHP_HASH_API void PHP_CRC32BEFinal(unsigned char digest[4], PHP_CRC32_CTX *context); PHP_HASH_API int PHP_CRC32Copy(const php_hash_ops *ops, PHP_CRC32_CTX *orig_context, PHP_CRC32_CTX *copy_context); #endif diff --git a/ext/hash/php_hash_crc32_tables.h b/ext/hash/php_hash_crc32_tables.h index 9d1e5b83765..8e2feccc018 100644 --- a/ext/hash/php_hash_crc32_tables.h +++ b/ext/hash/php_hash_crc32_tables.h @@ -136,3 +136,71 @@ static const uint32_t crc32b_table[] = { 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, }; + +static const uint32_t crc32c_table[] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351, +}; + diff --git a/ext/hash/tests/crc32.phpt b/ext/hash/tests/crc32.phpt index 80c6bb02863..f99152b49e0 100644 --- a/ext/hash/tests/crc32.phpt +++ b/ext/hash/tests/crc32.phpt @@ -2,6 +2,7 @@ Hash: CRC32 algorithm --FILE-- ?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"), "\n"; + ?> --EXPECT-- +crc32 00000000 6b9b9319 73bb8c64 @@ -25,6 +70,7 @@ echo hash('crc32b', '12345678901234567890123456789012345678901234567890123456789 9693bf77 882174a0 96790816 +crc32b 00000000 e8b7be43 352441c2 @@ -32,3 +78,42 @@ e8b7be43 4c2750bd 1fc2e6d2 7ca94a72 +crc32c +00000000 +c1d04330 +e2a22936 +364b3fb7 +92c80a31 +c450d697 +53bceff1 +e627f441 +0a9421b7 +2ddc99fc +e6599437 +9ee6ef25 +a245d57d +477a6781 +02bd79d0 +5e405e93 +516ad412 +b2cc01fe +0e28207f +be93f964 +9e3be0c3 +f505ef04 +85d3dc82 +c5142380 +75eb77dd +91ebe9f7 +f0b1168e +572b74e2 +8a58a6d5 +9c426c50 +735400a4 +bec49c95 +a95a2079 +de2e65c5 +297a88ed +66ed1d8b +dcded527 +9c44184b \ No newline at end of file diff --git a/ext/hash/tests/hash-clone.phpt b/ext/hash/tests/hash-clone.phpt index 7229b2a23ef..d5d4205354d 100644 --- a/ext/hash/tests/hash-clone.phpt +++ b/ext/hash/tests/hash-clone.phpt @@ -125,6 +125,9 @@ string(8) "e5cfc160" string(6) "crc32b" string(8) "69147a4e" string(8) "69147a4e" +string(6) "crc32c" +string(8) "5e405e93" +string(8) "5e405e93" string(6) "fnv132" string(8) "98139504" string(8) "98139504" @@ -281,6 +284,9 @@ string(8) "59f8d3d2" string(6) "crc32b" string(8) "69147a4e" string(8) "3ee63999" +string(6) "crc32c" +string(8) "5e405e93" +string(8) "516ad412" string(6) "fnv132" string(8) "98139504" string(8) "59ad036f" diff --git a/ext/hash/tests/hash_algos.phpt b/ext/hash/tests/hash_algos.phpt index efbb714788a..87efdc92134 100644 --- a/ext/hash/tests/hash_algos.phpt +++ b/ext/hash/tests/hash_algos.phpt @@ -16,7 +16,7 @@ var_dump(hash_algos()); ===Done=== --EXPECTF-- *** Testing hash_algos() : basic functionality *** -array(52) { +array(53) { [%d]=> string(3) "md2" [%d]=> @@ -82,6 +82,8 @@ array(52) { [%d]=> string(6) "crc32b" [%d]=> + string(6) "crc32c" + [%d]=> string(6) "fnv132" [%d]=> string(7) "fnv1a32" diff --git a/ext/hash/tests/hash_copy_001.phpt b/ext/hash/tests/hash_copy_001.phpt index 95aaf14c05a..8cd620e26d3 100644 --- a/ext/hash/tests/hash_copy_001.phpt +++ b/ext/hash/tests/hash_copy_001.phpt @@ -125,6 +125,9 @@ string(8) "e5cfc160" string(6) "crc32b" string(8) "69147a4e" string(8) "69147a4e" +string(6) "crc32c" +string(8) "5e405e93" +string(8) "5e405e93" string(6) "fnv132" string(8) "98139504" string(8) "98139504" @@ -281,6 +284,9 @@ string(8) "59f8d3d2" string(6) "crc32b" string(8) "69147a4e" string(8) "3ee63999" +string(6) "crc32c" +string(8) "5e405e93" +string(8) "516ad412" string(6) "fnv132" string(8) "98139504" string(8) "59ad036f"