Merge #10821: Add SSE4 optimized SHA256

6b8d872 Protect SSE4 code behind a compile-time flag (Pieter Wuille)
fa9be90 Add selftest for SHA256 transform (Pieter Wuille)
c1ccb15 Add SSE4 based SHA256 (Pieter Wuille)
2991c91 Add SHA256 dispatcher (Pieter Wuille)
4d50f38 Support multi-block SHA256 transforms (Pieter Wuille)

Pull request description:

  This adds an SSE4 assembly version of the SHA256 transform by Intel, and uses it at run time if SSE4 instructions are available, and use a fallback C++ implementation otherwise. Nearly every x86_64 CPU supports SSE4. The feature is only enabled when compiled with `--enable-experimental-asm`.

  In order to avoid build dependencies and other complications, the original Intel YASM code was translated to GCC extended asm syntax.

  This gives around a 50% speedup on the SHA256 benchmark for me.

  It is based on an earlier patch by @laanwj, though only includes a single assembly version (for now), and removes the YASM dependency.

Tree-SHA512: d31c50695ceb45264291537b93c0d7497670be38edf021ca5402eaa7d4e1e0e1ae492326e28d4e93979d066168129e62d1825e0384b1b906d36f85d93dfcb43c

configure.ac

@@ -177,6 +177,16 @@ AC_ARG_ENABLE([glibc-back-compat],
   [use_glibc_compat=$enableval],
   [use_glibc_compat=no])
 
+AC_ARG_ENABLE([experimental-asm],
+  [AS_HELP_STRING([--enable-experimental-asm],
+  [Enable experimental assembly routines (default is no)])],
+  [experimental_asm=$enableval],
+  [experimental_asm=no])
+
+if test "x$experimental_asm" = xyes; then
+  AC_DEFINE(EXPERIMENTAL_ASM, 1, [Define this symbol to build in experimental assembly routines])
+fi
+
 AC_ARG_WITH([system-univalue],
   [AS_HELP_STRING([--with-system-univalue],
   [Build with system UniValue (default is no)])],
@@ -1162,6 +1172,7 @@ AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes])
 AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes])
 AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes])
 AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes])
+AM_CONDITIONAL([EXPERIMENTAL_ASM],[test x$experimental_asm = xyes])
 
 AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
 AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])

src/Makefile.am

@@ -267,6 +267,10 @@ crypto_libbitcoin_crypto_a_SOURCES = \
   crypto/sha512.cpp \
   crypto/sha512.h
 
+if EXPERIMENTAL_ASM
+crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp
+endif
+
 # consensus: shared between all executables that validate any consensus rules.
 libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
 libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)

src/bench/bench_bitcoin.cpp

@@ -4,6 +4,7 @@
 
 #include "bench.h"
 
+#include "crypto/sha256.h"
 #include "key.h"
 #include "validation.h"
 #include "util.h"
@@ -12,6 +13,7 @@
 int
 main(int argc, char** argv)
 {
+    SHA256AutoDetect();
     RandomInit();
     ECC_Start();
     SetupEnvironment();

src/crypto/sha256.cpp

@@ -3,10 +3,21 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "crypto/sha256.h"
-
 #include "crypto/common.h"
 
+#include <assert.h>
 #include <string.h>
+#include <atomic>
+
+#if defined(__x86_64__) || defined(__amd64__)
+#if defined(EXPERIMENTAL_ASM)
+#include <cpuid.h>
+namespace sha256_sse4
+{
+void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
+}
+#endif
+#endif
 
 // Internal implementation code.
 namespace
@@ -43,9 +54,10 @@ void inline Initialize(uint32_t* s)
     s[7] = 0x5be0cd19ul;
 }
 
-/** Perform one SHA-256 transformation, processing a 64-byte chunk. */
+/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */
-void Transform(uint32_t* s, const unsigned char* chunk)
+void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
 {
+    while (blocks--) {
         uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7];
         uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15;
 
@@ -125,11 +137,59 @@ void Transform(uint32_t* s, const unsigned char* chunk)
         s[5] += f;
         s[6] += g;
         s[7] += h;
+        chunk += 64;
+    }
 }
 
 } // namespace sha256
+
+typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t);
+
+bool SelfTest(TransformType tr) {
+    static const unsigned char in1[65] = {0, 0x80};
+    static const unsigned char in2[129] = {
+        0,
+        32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 
+        32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 
+        0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0
+    };
+    static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul};
+    static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul};
+    static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul};
+    uint32_t buf[8];
+    memcpy(buf, init, sizeof(buf));
+    // Process nothing, and check we remain in the initial state.
+    tr(buf, nullptr, 0);
+    if (memcmp(buf, init, sizeof(buf))) return false;
+    // Process the padded empty string (unaligned)
+    tr(buf, in1 + 1, 1);
+    if (memcmp(buf, out1, sizeof(buf))) return false;
+    // Process 64 spaces (unaligned)
+    memcpy(buf, init, sizeof(buf));
+    tr(buf, in2 + 1, 2);
+    if (memcmp(buf, out2, sizeof(buf))) return false;
+    return true;
+}
+
+TransformType Transform = sha256::Transform;
+
 } // namespace
 
+std::string SHA256AutoDetect()
+{
+#if defined(EXPERIMENTAL_ASM) && (defined(__x86_64__) || defined(__amd64__))
+    uint32_t eax, ebx, ecx, edx;
+    if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) {
+        Transform = sha256_sse4::Transform;
+        assert(SelfTest(Transform));
+        return "sse4";
+    }
+#endif
+
+    assert(SelfTest(Transform));
+    return "standard";
+}
 
 ////// SHA-256
 
@@ -147,14 +207,14 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len)
         memcpy(buf + bufsize, data, 64 - bufsize);
         bytes += 64 - bufsize;
         data += 64 - bufsize;
-        sha256::Transform(s, buf);
+        Transform(s, buf, 1);
         bufsize = 0;
     }
-    while (end >= data + 64) {
+    if (end - data >= 64) {
-        // Process full chunks directly from the source.
+        size_t blocks = (end - data) / 64;
-        sha256::Transform(s, data);
+        Transform(s, data, blocks);
-        bytes += 64;
+        data += 64 * blocks;
-        data += 64;
+        bytes += 64 * blocks;
     }
     if (end > data) {
         // Fill the buffer with what remains.

src/crypto/sha256.h

@@ -7,6 +7,7 @@
 
 #include <stdint.h>
 #include <stdlib.h>
+#include <string>
 
 /** A hasher class for SHA-256. */
 class CSHA256
@@ -25,4 +26,9 @@ public:
     CSHA256& Reset();
 };
 
+/** Autodetect the best available SHA256 implementation.
+ *  Returns the name of the implementation.
+ */
+std::string SHA256AutoDetect();
+
 #endif // BITCOIN_CRYPTO_SHA256_H

src/init.cpp

@@ -1161,6 +1161,8 @@ bool AppInitSanityChecks()
     // ********************************************************* Step 4: sanity checks
 
     // Initialize elliptic curve code
+    std::string sha256_algo = SHA256AutoDetect();
+    LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo);
     RandomInit();
     ECC_Start();
     globalVerifyHandle.reset(new ECCVerifyHandle());

src/test/test_bitcoin.cpp

@@ -7,6 +7,7 @@
 #include "chainparams.h"
 #include "consensus/consensus.h"
 #include "consensus/validation.h"
+#include "crypto/sha256.h"
 #include "fs.h"
 #include "key.h"
 #include "validation.h"
@@ -33,6 +34,7 @@ extern void noui_connect();
 
 BasicTestingSetup::BasicTestingSetup(const std::string& chainName)
 {
+        SHA256AutoDetect();
         RandomInit();
         ECC_Start();
         SetupEnvironment();