From 2b0be077600beb95066ffa5f06094eeca906851a Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Fri, 18 Apr 2025 19:06:16 -0400 Subject: [PATCH] Add code from BearSSL's ghash implementation. Polyval (which we need for CGO) is very similar to ghash, and most of this code should be reusable with suitable adaptation. --- LICENSE | 27 +++ src/ext/polyval/README.tor | 28 +++ src/ext/polyval/ghash_ctmul.c | 345 ++++++++++++++++++++++++++++ src/ext/polyval/ghash_ctmul64.c | 154 +++++++++++++ src/ext/polyval/ghash_pclmul.c | 389 ++++++++++++++++++++++++++++++++ 5 files changed, 943 insertions(+) create mode 100644 src/ext/polyval/README.tor create mode 100644 src/ext/polyval/ghash_ctmul.c create mode 100644 src/ext/polyval/ghash_ctmul64.c create mode 100644 src/ext/polyval/ghash_pclmul.c diff --git a/LICENSE b/LICENSE index abab8e33fe..57e22aadb4 100644 --- a/LICENSE +++ b/LICENSE @@ -382,6 +382,33 @@ src/ext/mulodi4.c is distributed under this license: names of the LLVM Team or the University of Illinois to endorse or promote products derived from this Software. +=============================================================================== +Parts of src/ext/polyval are based on Thomas Pornin's GHASH implementation in +BearSSL, and distributed under the following license: + + Copyright (c) 2016 Thomas Pornin + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + =============================================================================== If you got Tor as a static binary with OpenSSL included, then you should know: "This product includes software developed by the OpenSSL Project diff --git a/src/ext/polyval/README.tor b/src/ext/polyval/README.tor new file mode 100644 index 0000000000..34533abfb0 --- /dev/null +++ b/src/ext/polyval/README.tor @@ -0,0 +1,28 @@ +This code is based on the constant-time GHASH implementations +from BearSSL, written by Thomas Pornin. + +Up-to-date with BearSSL as of 3c040368f6791553610e362401db1efff4b4c5b8. + +The license on that code is: + +> Copyright (c) 2016 Thomas Pornin +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +> BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +> ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +> CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. diff --git a/src/ext/polyval/ghash_ctmul.c b/src/ext/polyval/ghash_ctmul.c new file mode 100644 index 0000000000..362320252f --- /dev/null +++ b/src/ext/polyval/ghash_ctmul.c @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2016 Thomas Pornin + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "inner.h" + +/* + * We compute "carryless multiplications" through normal integer + * multiplications, masking out enough bits to create "holes" in which + * carries may expand without altering our bits; we really use 8 data + * bits per 32-bit word, spaced every fourth bit. Accumulated carries + * may not exceed 8 in total, which fits in 4 bits. + * + * It would be possible to use a 3-bit spacing, allowing two operands, + * one with 7 non-zero data bits, the other one with 10 or 11 non-zero + * data bits; this asymmetric splitting makes the overall code more + * complex with thresholds and exceptions, and does not appear to be + * worth the effort. + */ + +/* + * We cannot really autodetect whether multiplications are "slow" or + * not. A typical example is the ARM Cortex M0+, which exists in two + * versions: one with a 1-cycle multiplication opcode, the other with + * a 32-cycle multiplication opcode. They both use exactly the same + * architecture and ABI, and cannot be distinguished from each other + * at compile-time. + * + * Since most modern CPU (even embedded CPU) still have fast + * multiplications, we use the "fast mul" code by default. + */ + +#if BR_SLOW_MUL + +/* + * This implementation uses Karatsuba-like reduction to make fewer + * integer multiplications (9 instead of 16), at the expense of extra + * logical operations (XOR, shifts...). On modern x86 CPU that offer + * fast, pipelined multiplications, this code is about twice slower than + * the simpler code with 16 multiplications. This tendency may be + * reversed on low-end platforms with expensive multiplications. + */ + +#define MUL32(h, l, x, y) do { \ + uint64_t mul32tmp = MUL(x, y); \ + (h) = (uint32_t)(mul32tmp >> 32); \ + (l) = (uint32_t)mul32tmp; \ + } while (0) + +static inline void +bmul(uint32_t *hi, uint32_t *lo, uint32_t x, uint32_t y) +{ + uint32_t x0, x1, x2, x3; + uint32_t y0, y1, y2, y3; + uint32_t a0, a1, a2, a3, a4, a5, a6, a7, a8; + uint32_t b0, b1, b2, b3, b4, b5, b6, b7, b8; + + x0 = x & (uint32_t)0x11111111; + x1 = x & (uint32_t)0x22222222; + x2 = x & (uint32_t)0x44444444; + x3 = x & (uint32_t)0x88888888; + y0 = y & (uint32_t)0x11111111; + y1 = y & (uint32_t)0x22222222; + y2 = y & (uint32_t)0x44444444; + y3 = y & (uint32_t)0x88888888; + + /* + * (x0+W*x1)*(y0+W*y1) -> a0:b0 + * (x2+W*x3)*(y2+W*y3) -> a3:b3 + * ((x0+x2)+W*(x1+x3))*((y0+y2)+W*(y1+y3)) -> a6:b6 + */ + a0 = x0; + b0 = y0; + a1 = x1 >> 1; + b1 = y1 >> 1; + a2 = a0 ^ a1; + b2 = b0 ^ b1; + a3 = x2 >> 2; + b3 = y2 >> 2; + a4 = x3 >> 3; + b4 = y3 >> 3; + a5 = a3 ^ a4; + b5 = b3 ^ b4; + a6 = a0 ^ a3; + b6 = b0 ^ b3; + a7 = a1 ^ a4; + b7 = b1 ^ b4; + a8 = a6 ^ a7; + b8 = b6 ^ b7; + + MUL32(b0, a0, b0, a0); + MUL32(b1, a1, b1, a1); + MUL32(b2, a2, b2, a2); + MUL32(b3, a3, b3, a3); + MUL32(b4, a4, b4, a4); + MUL32(b5, a5, b5, a5); + MUL32(b6, a6, b6, a6); + MUL32(b7, a7, b7, a7); + MUL32(b8, a8, b8, a8); + + a0 &= (uint32_t)0x11111111; + a1 &= (uint32_t)0x11111111; + a2 &= (uint32_t)0x11111111; + a3 &= (uint32_t)0x11111111; + a4 &= (uint32_t)0x11111111; + a5 &= (uint32_t)0x11111111; + a6 &= (uint32_t)0x11111111; + a7 &= (uint32_t)0x11111111; + a8 &= (uint32_t)0x11111111; + b0 &= (uint32_t)0x11111111; + b1 &= (uint32_t)0x11111111; + b2 &= (uint32_t)0x11111111; + b3 &= (uint32_t)0x11111111; + b4 &= (uint32_t)0x11111111; + b5 &= (uint32_t)0x11111111; + b6 &= (uint32_t)0x11111111; + b7 &= (uint32_t)0x11111111; + b8 &= (uint32_t)0x11111111; + + a2 ^= a0 ^ a1; + b2 ^= b0 ^ b1; + a0 ^= (a2 << 1) ^ (a1 << 2); + b0 ^= (b2 << 1) ^ (b1 << 2); + a5 ^= a3 ^ a4; + b5 ^= b3 ^ b4; + a3 ^= (a5 << 1) ^ (a4 << 2); + b3 ^= (b5 << 1) ^ (b4 << 2); + a8 ^= a6 ^ a7; + b8 ^= b6 ^ b7; + a6 ^= (a8 << 1) ^ (a7 << 2); + b6 ^= (b8 << 1) ^ (b7 << 2); + a6 ^= a0 ^ a3; + b6 ^= b0 ^ b3; + *lo = a0 ^ (a6 << 2) ^ (a3 << 4); + *hi = b0 ^ (b6 << 2) ^ (b3 << 4) ^ (a6 >> 30) ^ (a3 >> 28); +} + +#else + +/* + * Simple multiplication in GF(2)[X], using 16 integer multiplications. + */ + +static inline void +bmul(uint32_t *hi, uint32_t *lo, uint32_t x, uint32_t y) +{ + uint32_t x0, x1, x2, x3; + uint32_t y0, y1, y2, y3; + uint64_t z0, z1, z2, z3; + uint64_t z; + + x0 = x & (uint32_t)0x11111111; + x1 = x & (uint32_t)0x22222222; + x2 = x & (uint32_t)0x44444444; + x3 = x & (uint32_t)0x88888888; + y0 = y & (uint32_t)0x11111111; + y1 = y & (uint32_t)0x22222222; + y2 = y & (uint32_t)0x44444444; + y3 = y & (uint32_t)0x88888888; + z0 = MUL(x0, y0) ^ MUL(x1, y3) ^ MUL(x2, y2) ^ MUL(x3, y1); + z1 = MUL(x0, y1) ^ MUL(x1, y0) ^ MUL(x2, y3) ^ MUL(x3, y2); + z2 = MUL(x0, y2) ^ MUL(x1, y1) ^ MUL(x2, y0) ^ MUL(x3, y3); + z3 = MUL(x0, y3) ^ MUL(x1, y2) ^ MUL(x2, y1) ^ MUL(x3, y0); + z0 &= (uint64_t)0x1111111111111111; + z1 &= (uint64_t)0x2222222222222222; + z2 &= (uint64_t)0x4444444444444444; + z3 &= (uint64_t)0x8888888888888888; + z = z0 | z1 | z2 | z3; + *lo = (uint32_t)z; + *hi = (uint32_t)(z >> 32); +} + +#endif + +/* see bearssl_hash.h */ +void +br_ghash_ctmul(void *y, const void *h, const void *data, size_t len) +{ + const unsigned char *buf, *hb; + unsigned char *yb; + uint32_t yw[4]; + uint32_t hw[4]; + + /* + * Throughout the loop we handle the y and h values as arrays + * of 32-bit words. + */ + buf = data; + yb = y; + hb = h; + yw[3] = br_dec32be(yb); + yw[2] = br_dec32be(yb + 4); + yw[1] = br_dec32be(yb + 8); + yw[0] = br_dec32be(yb + 12); + hw[3] = br_dec32be(hb); + hw[2] = br_dec32be(hb + 4); + hw[1] = br_dec32be(hb + 8); + hw[0] = br_dec32be(hb + 12); + while (len > 0) { + const unsigned char *src; + unsigned char tmp[16]; + int i; + uint32_t a[9], b[9], zw[8]; + uint32_t c0, c1, c2, c3, d0, d1, d2, d3, e0, e1, e2, e3; + + /* + * Get the next 16-byte block (using zero-padding if + * necessary). + */ + if (len >= 16) { + src = buf; + buf += 16; + len -= 16; + } else { + memcpy(tmp, buf, len); + memset(tmp + len, 0, (sizeof tmp) - len); + src = tmp; + len = 0; + } + + /* + * Decode the block. The GHASH standard mandates + * big-endian encoding. + */ + yw[3] ^= br_dec32be(src); + yw[2] ^= br_dec32be(src + 4); + yw[1] ^= br_dec32be(src + 8); + yw[0] ^= br_dec32be(src + 12); + + /* + * We multiply two 128-bit field elements. We use + * Karatsuba to turn that into three 64-bit + * multiplications, which are themselves done with a + * total of nine 32-bit multiplications. + */ + + /* + * y[0,1]*h[0,1] -> 0..2 + * y[2,3]*h[2,3] -> 3..5 + * (y[0,1]+y[2,3])*(h[0,1]+h[2,3]) -> 6..8 + */ + a[0] = yw[0]; + b[0] = hw[0]; + a[1] = yw[1]; + b[1] = hw[1]; + a[2] = a[0] ^ a[1]; + b[2] = b[0] ^ b[1]; + + a[3] = yw[2]; + b[3] = hw[2]; + a[4] = yw[3]; + b[4] = hw[3]; + a[5] = a[3] ^ a[4]; + b[5] = b[3] ^ b[4]; + + a[6] = a[0] ^ a[3]; + b[6] = b[0] ^ b[3]; + a[7] = a[1] ^ a[4]; + b[7] = b[1] ^ b[4]; + a[8] = a[6] ^ a[7]; + b[8] = b[6] ^ b[7]; + + for (i = 0; i < 9; i ++) { + bmul(&b[i], &a[i], b[i], a[i]); + } + + c0 = a[0]; + c1 = b[0] ^ a[2] ^ a[0] ^ a[1]; + c2 = a[1] ^ b[2] ^ b[0] ^ b[1]; + c3 = b[1]; + d0 = a[3]; + d1 = b[3] ^ a[5] ^ a[3] ^ a[4]; + d2 = a[4] ^ b[5] ^ b[3] ^ b[4]; + d3 = b[4]; + e0 = a[6]; + e1 = b[6] ^ a[8] ^ a[6] ^ a[7]; + e2 = a[7] ^ b[8] ^ b[6] ^ b[7]; + e3 = b[7]; + + e0 ^= c0 ^ d0; + e1 ^= c1 ^ d1; + e2 ^= c2 ^ d2; + e3 ^= c3 ^ d3; + c2 ^= e0; + c3 ^= e1; + d0 ^= e2; + d1 ^= e3; + + /* + * GHASH specification has the bits "reversed" (most + * significant is in fact least significant), which does + * not matter for a carryless multiplication, except that + * the 255-bit result must be shifted by 1 bit. + */ + zw[0] = c0 << 1; + zw[1] = (c1 << 1) | (c0 >> 31); + zw[2] = (c2 << 1) | (c1 >> 31); + zw[3] = (c3 << 1) | (c2 >> 31); + zw[4] = (d0 << 1) | (c3 >> 31); + zw[5] = (d1 << 1) | (d0 >> 31); + zw[6] = (d2 << 1) | (d1 >> 31); + zw[7] = (d3 << 1) | (d2 >> 31); + + /* + * We now do the reduction modulo the field polynomial + * to get back to 128 bits. + */ + for (i = 0; i < 4; i ++) { + uint32_t lw; + + lw = zw[i]; + zw[i + 4] ^= lw ^ (lw >> 1) ^ (lw >> 2) ^ (lw >> 7); + zw[i + 3] ^= (lw << 31) ^ (lw << 30) ^ (lw << 25); + } + memcpy(yw, zw + 4, sizeof yw); + } + + /* + * Encode back the result. + */ + br_enc32be(yb, yw[3]); + br_enc32be(yb + 4, yw[2]); + br_enc32be(yb + 8, yw[1]); + br_enc32be(yb + 12, yw[0]); +} diff --git a/src/ext/polyval/ghash_ctmul64.c b/src/ext/polyval/ghash_ctmul64.c new file mode 100644 index 0000000000..a46f16fee9 --- /dev/null +++ b/src/ext/polyval/ghash_ctmul64.c @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2016 Thomas Pornin + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "inner.h" + +/* + * This is the 64-bit variant of br_ghash_ctmul32(), with 64-bit operands + * and bit reversal of 64-bit words. + */ + +static inline uint64_t +bmul64(uint64_t x, uint64_t y) +{ + uint64_t x0, x1, x2, x3; + uint64_t y0, y1, y2, y3; + uint64_t z0, z1, z2, z3; + + x0 = x & (uint64_t)0x1111111111111111; + x1 = x & (uint64_t)0x2222222222222222; + x2 = x & (uint64_t)0x4444444444444444; + x3 = x & (uint64_t)0x8888888888888888; + y0 = y & (uint64_t)0x1111111111111111; + y1 = y & (uint64_t)0x2222222222222222; + y2 = y & (uint64_t)0x4444444444444444; + y3 = y & (uint64_t)0x8888888888888888; + z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1); + z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2); + z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3); + z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0); + z0 &= (uint64_t)0x1111111111111111; + z1 &= (uint64_t)0x2222222222222222; + z2 &= (uint64_t)0x4444444444444444; + z3 &= (uint64_t)0x8888888888888888; + return z0 | z1 | z2 | z3; +} + +static uint64_t +rev64(uint64_t x) +{ +#define RMS(m, s) do { \ + x = ((x & (uint64_t)(m)) << (s)) \ + | ((x >> (s)) & (uint64_t)(m)); \ + } while (0) + + RMS(0x5555555555555555, 1); + RMS(0x3333333333333333, 2); + RMS(0x0F0F0F0F0F0F0F0F, 4); + RMS(0x00FF00FF00FF00FF, 8); + RMS(0x0000FFFF0000FFFF, 16); + return (x << 32) | (x >> 32); + +#undef RMS +} + +/* see bearssl_ghash.h */ +void +br_ghash_ctmul64(void *y, const void *h, const void *data, size_t len) +{ + const unsigned char *buf, *hb; + unsigned char *yb; + uint64_t y0, y1; + uint64_t h0, h1, h2, h0r, h1r, h2r; + + buf = data; + yb = y; + hb = h; + y1 = br_dec64be(yb); + y0 = br_dec64be(yb + 8); + h1 = br_dec64be(hb); + h0 = br_dec64be(hb + 8); + h0r = rev64(h0); + h1r = rev64(h1); + h2 = h0 ^ h1; + h2r = h0r ^ h1r; + while (len > 0) { + const unsigned char *src; + unsigned char tmp[16]; + uint64_t y0r, y1r, y2, y2r; + uint64_t z0, z1, z2, z0h, z1h, z2h; + uint64_t v0, v1, v2, v3; + + if (len >= 16) { + src = buf; + buf += 16; + len -= 16; + } else { + memcpy(tmp, buf, len); + memset(tmp + len, 0, (sizeof tmp) - len); + src = tmp; + len = 0; + } + y1 ^= br_dec64be(src); + y0 ^= br_dec64be(src + 8); + + y0r = rev64(y0); + y1r = rev64(y1); + y2 = y0 ^ y1; + y2r = y0r ^ y1r; + + z0 = bmul64(y0, h0); + z1 = bmul64(y1, h1); + z2 = bmul64(y2, h2); + z0h = bmul64(y0r, h0r); + z1h = bmul64(y1r, h1r); + z2h = bmul64(y2r, h2r); + z2 ^= z0 ^ z1; + z2h ^= z0h ^ z1h; + z0h = rev64(z0h) >> 1; + z1h = rev64(z1h) >> 1; + z2h = rev64(z2h) >> 1; + + v0 = z0; + v1 = z0h ^ z2; + v2 = z1 ^ z2h; + v3 = z1h; + + v3 = (v3 << 1) | (v2 >> 63); + v2 = (v2 << 1) | (v1 >> 63); + v1 = (v1 << 1) | (v0 >> 63); + v0 = (v0 << 1); + + v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7); + v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57); + v3 ^= v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7); + v2 ^= (v1 << 63) ^ (v1 << 62) ^ (v1 << 57); + + y0 = v2; + y1 = v3; + } + + br_enc64be(yb, y1); + br_enc64be(yb + 8, y0); +} diff --git a/src/ext/polyval/ghash_pclmul.c b/src/ext/polyval/ghash_pclmul.c new file mode 100644 index 0000000000..a58e7dc02b --- /dev/null +++ b/src/ext/polyval/ghash_pclmul.c @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2017 Thomas Pornin + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#define BR_ENABLE_INTRINSICS 1 +#include "inner.h" + +/* + * This is the GHASH implementation that leverages the pclmulqdq opcode + * (from the AES-NI instructions). + */ + +#if BR_AES_X86NI + +/* + * Test CPU support for PCLMULQDQ. + */ +static inline int +pclmul_supported(void) +{ + /* + * Bit mask for features in ECX: + * 1 PCLMULQDQ support + */ + return br_cpuid(0, 0, 0x00000002, 0); +} + +/* see bearssl_hash.h */ +br_ghash +br_ghash_pclmul_get(void) +{ + return pclmul_supported() ? &br_ghash_pclmul : 0; +} + +BR_TARGETS_X86_UP + +/* + * GHASH is defined over elements of GF(2^128) with "full little-endian" + * representation: leftmost byte is least significant, and, within each + * byte, leftmost _bit_ is least significant. The natural ordering in + * x86 is "mixed little-endian": bytes are ordered from least to most + * significant, but bits within a byte are in most-to-least significant + * order. Going to full little-endian representation would require + * reversing bits within each byte, which is doable but expensive. + * + * Instead, we go to full big-endian representation, by swapping bytes + * around, which is done with a single _mm_shuffle_epi8() opcode (it + * comes with SSSE3; all CPU that offer pclmulqdq also have SSSE3). We + * can use a full big-endian representation because in a carryless + * multiplication, we have a nice bit reversal property: + * + * rev_128(x) * rev_128(y) = rev_255(x * y) + * + * So by using full big-endian, we still get the right result, except + * that it is right-shifted by 1 bit. The left-shift is relatively + * inexpensive, and it can be mutualised. + * + * + * Since SSE2 opcodes do not have facilities for shitfting full 128-bit + * values with bit precision, we have to break down values into 64-bit + * chunks. We number chunks from 0 to 3 in left to right order. + */ + +/* + * Byte-swap a complete 128-bit value. This normally uses + * _mm_shuffle_epi8(), which gets translated to pshufb (an SSSE3 opcode). + * However, this crashes old Clang versions, so, for Clang before 3.8, + * we use an alternate (and less efficient) version. + */ +#if BR_CLANG && !BR_CLANG_3_8 +#define BYTESWAP_DECL +#define BYTESWAP_PREP (void)0 +#define BYTESWAP(x) do { \ + __m128i byteswap1, byteswap2; \ + byteswap1 = (x); \ + byteswap2 = _mm_srli_epi16(byteswap1, 8); \ + byteswap1 = _mm_slli_epi16(byteswap1, 8); \ + byteswap1 = _mm_or_si128(byteswap1, byteswap2); \ + byteswap1 = _mm_shufflelo_epi16(byteswap1, 0x1B); \ + byteswap1 = _mm_shufflehi_epi16(byteswap1, 0x1B); \ + (x) = _mm_shuffle_epi32(byteswap1, 0x4E); \ + } while (0) +#else +#define BYTESWAP_DECL __m128i byteswap_index; +#define BYTESWAP_PREP do { \ + byteswap_index = _mm_set_epi8( \ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); \ + } while (0) +#define BYTESWAP(x) do { \ + (x) = _mm_shuffle_epi8((x), byteswap_index); \ + } while (0) +#endif + +/* + * Call pclmulqdq. Clang appears to have trouble with the intrinsic, so, + * for that compiler, we use inline assembly. Inline assembly is + * potentially a bit slower because the compiler does not understand + * what the opcode does, and thus cannot optimize instruction + * scheduling. + * + * We use a target of "sse2" only, so that Clang may still handle the + * '__m128i' type and allocate SSE2 registers. + */ +#if BR_CLANG +BR_TARGET("sse2") +static inline __m128i +pclmulqdq00(__m128i x, __m128i y) +{ + __asm__ ("pclmulqdq $0x00, %1, %0" : "+x" (x) : "x" (y)); + return x; +} +BR_TARGET("sse2") +static inline __m128i +pclmulqdq11(__m128i x, __m128i y) +{ + __asm__ ("pclmulqdq $0x11, %1, %0" : "+x" (x) : "x" (y)); + return x; +} +#else +#define pclmulqdq00(x, y) _mm_clmulepi64_si128(x, y, 0x00) +#define pclmulqdq11(x, y) _mm_clmulepi64_si128(x, y, 0x11) +#endif + +/* + * From a 128-bit value kw, compute kx as the XOR of the two 64-bit + * halves of kw (into the right half of kx; left half is unspecified). + */ +#define BK(kw, kx) do { \ + kx = _mm_xor_si128(kw, _mm_shuffle_epi32(kw, 0x0E)); \ + } while (0) + +/* + * Combine two 64-bit values (k0:k1) into a 128-bit (kw) value and + * the XOR of the two values (kx). + */ +#define PBK(k0, k1, kw, kx) do { \ + kw = _mm_unpacklo_epi64(k1, k0); \ + kx = _mm_xor_si128(k0, k1); \ + } while (0) + +/* + * Left-shift by 1 bit a 256-bit value (in four 64-bit words). + */ +#define SL_256(x0, x1, x2, x3) do { \ + x0 = _mm_or_si128( \ + _mm_slli_epi64(x0, 1), \ + _mm_srli_epi64(x1, 63)); \ + x1 = _mm_or_si128( \ + _mm_slli_epi64(x1, 1), \ + _mm_srli_epi64(x2, 63)); \ + x2 = _mm_or_si128( \ + _mm_slli_epi64(x2, 1), \ + _mm_srli_epi64(x3, 63)); \ + x3 = _mm_slli_epi64(x3, 1); \ + } while (0) + +/* + * Perform reduction in GF(2^128). The 256-bit value is in x0..x3; + * result is written in x0..x1. + */ +#define REDUCE_F128(x0, x1, x2, x3) do { \ + x1 = _mm_xor_si128( \ + x1, \ + _mm_xor_si128( \ + _mm_xor_si128( \ + x3, \ + _mm_srli_epi64(x3, 1)), \ + _mm_xor_si128( \ + _mm_srli_epi64(x3, 2), \ + _mm_srli_epi64(x3, 7)))); \ + x2 = _mm_xor_si128( \ + _mm_xor_si128( \ + x2, \ + _mm_slli_epi64(x3, 63)), \ + _mm_xor_si128( \ + _mm_slli_epi64(x3, 62), \ + _mm_slli_epi64(x3, 57))); \ + x0 = _mm_xor_si128( \ + x0, \ + _mm_xor_si128( \ + _mm_xor_si128( \ + x2, \ + _mm_srli_epi64(x2, 1)), \ + _mm_xor_si128( \ + _mm_srli_epi64(x2, 2), \ + _mm_srli_epi64(x2, 7)))); \ + x1 = _mm_xor_si128( \ + _mm_xor_si128( \ + x1, \ + _mm_slli_epi64(x2, 63)), \ + _mm_xor_si128( \ + _mm_slli_epi64(x2, 62), \ + _mm_slli_epi64(x2, 57))); \ + } while (0) + +/* + * Square value kw into (dw,dx). + */ +#define SQUARE_F128(kw, dw, dx) do { \ + __m128i z0, z1, z2, z3; \ + z1 = pclmulqdq11(kw, kw); \ + z3 = pclmulqdq00(kw, kw); \ + z0 = _mm_shuffle_epi32(z1, 0x0E); \ + z2 = _mm_shuffle_epi32(z3, 0x0E); \ + SL_256(z0, z1, z2, z3); \ + REDUCE_F128(z0, z1, z2, z3); \ + PBK(z0, z1, dw, dx); \ + } while (0) + +/* see bearssl_hash.h */ +BR_TARGET("ssse3,pclmul") +void +br_ghash_pclmul(void *y, const void *h, const void *data, size_t len) +{ + const unsigned char *buf1, *buf2; + unsigned char tmp[64]; + size_t num4, num1; + __m128i yw, h1w, h1x; + BYTESWAP_DECL + + /* + * We split data into two chunks. First chunk starts at buf1 + * and contains num4 blocks of 64-byte values. Second chunk + * starts at buf2 and contains num1 blocks of 16-byte values. + * We want the first chunk to be as large as possible. + */ + buf1 = data; + num4 = len >> 6; + len &= 63; + buf2 = buf1 + (num4 << 6); + num1 = (len + 15) >> 4; + if ((len & 15) != 0) { + memcpy(tmp, buf2, len); + memset(tmp + len, 0, (num1 << 4) - len); + buf2 = tmp; + } + + /* + * Preparatory step for endian conversions. + */ + BYTESWAP_PREP; + + /* + * Load y and h. + */ + yw = _mm_loadu_si128(y); + h1w = _mm_loadu_si128(h); + BYTESWAP(yw); + BYTESWAP(h1w); + BK(h1w, h1x); + + if (num4 > 0) { + __m128i h2w, h2x, h3w, h3x, h4w, h4x; + __m128i t0, t1, t2, t3; + + /* + * Compute h2 = h^2. + */ + SQUARE_F128(h1w, h2w, h2x); + + /* + * Compute h3 = h^3 = h*(h^2). + */ + t1 = pclmulqdq11(h1w, h2w); + t3 = pclmulqdq00(h1w, h2w); + t2 = _mm_xor_si128(pclmulqdq00(h1x, h2x), + _mm_xor_si128(t1, t3)); + t0 = _mm_shuffle_epi32(t1, 0x0E); + t1 = _mm_xor_si128(t1, _mm_shuffle_epi32(t2, 0x0E)); + t2 = _mm_xor_si128(t2, _mm_shuffle_epi32(t3, 0x0E)); + SL_256(t0, t1, t2, t3); + REDUCE_F128(t0, t1, t2, t3); + PBK(t0, t1, h3w, h3x); + + /* + * Compute h4 = h^4 = (h^2)^2. + */ + SQUARE_F128(h2w, h4w, h4x); + + while (num4 -- > 0) { + __m128i aw0, aw1, aw2, aw3; + __m128i ax0, ax1, ax2, ax3; + + aw0 = _mm_loadu_si128((void *)(buf1 + 0)); + aw1 = _mm_loadu_si128((void *)(buf1 + 16)); + aw2 = _mm_loadu_si128((void *)(buf1 + 32)); + aw3 = _mm_loadu_si128((void *)(buf1 + 48)); + BYTESWAP(aw0); + BYTESWAP(aw1); + BYTESWAP(aw2); + BYTESWAP(aw3); + buf1 += 64; + + aw0 = _mm_xor_si128(aw0, yw); + BK(aw1, ax1); + BK(aw2, ax2); + BK(aw3, ax3); + BK(aw0, ax0); + + t1 = _mm_xor_si128( + _mm_xor_si128( + pclmulqdq11(aw0, h4w), + pclmulqdq11(aw1, h3w)), + _mm_xor_si128( + pclmulqdq11(aw2, h2w), + pclmulqdq11(aw3, h1w))); + t3 = _mm_xor_si128( + _mm_xor_si128( + pclmulqdq00(aw0, h4w), + pclmulqdq00(aw1, h3w)), + _mm_xor_si128( + pclmulqdq00(aw2, h2w), + pclmulqdq00(aw3, h1w))); + t2 = _mm_xor_si128( + _mm_xor_si128( + pclmulqdq00(ax0, h4x), + pclmulqdq00(ax1, h3x)), + _mm_xor_si128( + pclmulqdq00(ax2, h2x), + pclmulqdq00(ax3, h1x))); + t2 = _mm_xor_si128(t2, _mm_xor_si128(t1, t3)); + t0 = _mm_shuffle_epi32(t1, 0x0E); + t1 = _mm_xor_si128(t1, _mm_shuffle_epi32(t2, 0x0E)); + t2 = _mm_xor_si128(t2, _mm_shuffle_epi32(t3, 0x0E)); + SL_256(t0, t1, t2, t3); + REDUCE_F128(t0, t1, t2, t3); + yw = _mm_unpacklo_epi64(t1, t0); + } + } + + while (num1 -- > 0) { + __m128i aw, ax; + __m128i t0, t1, t2, t3; + + aw = _mm_loadu_si128((void *)buf2); + BYTESWAP(aw); + buf2 += 16; + + aw = _mm_xor_si128(aw, yw); + BK(aw, ax); + + t1 = pclmulqdq11(aw, h1w); + t3 = pclmulqdq00(aw, h1w); + t2 = pclmulqdq00(ax, h1x); + t2 = _mm_xor_si128(t2, _mm_xor_si128(t1, t3)); + t0 = _mm_shuffle_epi32(t1, 0x0E); + t1 = _mm_xor_si128(t1, _mm_shuffle_epi32(t2, 0x0E)); + t2 = _mm_xor_si128(t2, _mm_shuffle_epi32(t3, 0x0E)); + SL_256(t0, t1, t2, t3); + REDUCE_F128(t0, t1, t2, t3); + yw = _mm_unpacklo_epi64(t1, t0); + } + + BYTESWAP(yw); + _mm_storeu_si128(y, yw); +} + +BR_TARGETS_X86_DOWN + +#else + +/* see bearssl_hash.h */ +br_ghash +br_ghash_pclmul_get(void) +{ + return 0; +} + +#endif