Changeset View
Changeset View
Standalone View
Standalone View
src/crypto/sha256.cpp
// Copyright (c) 2014 The Bitcoin Core developers | // Copyright (c) 2014 The Bitcoin Core developers | ||||
// Distributed under the MIT software license, see the accompanying | // Distributed under the MIT software license, see the accompanying | ||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php. | // file COPYING or http://www.opensource.org/licenses/mit-license.php. | ||||
#include "crypto/sha256.h" | #include "crypto/sha256.h" | ||||
#include "crypto/common.h" | #include "crypto/common.h" | ||||
#include <atomic> | #include <atomic> | ||||
#include <cassert> | #include <cassert> | ||||
#include <cstring> | #include <cstring> | ||||
#if defined(__x86_64__) || defined(__amd64__) | #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) | ||||
#if defined(USE_ASM) | #if defined(USE_ASM) | ||||
#include <cpuid.h> | #include <cpuid.h> | ||||
namespace sha256_sse4 { | namespace sha256_sse4 { | ||||
void Transform(uint32_t *s, const uint8_t *chunk, size_t blocks); | void Transform(uint32_t *s, const uint8_t *chunk, size_t blocks); | ||||
} | } | ||||
#endif | #endif | ||||
#endif | #endif | ||||
namespace sha256d64_sse41 { | namespace sha256d64_sse41 { | ||||
void Transform_4way(uint8_t *out, const uint8_t *in); | void Transform_4way(uint8_t *out, const uint8_t *in); | ||||
} | } | ||||
namespace sha256d64_avx2 { | namespace sha256d64_avx2 { | ||||
void Transform_8way(uint8_t *out, const uint8_t *in); | void Transform_8way(uint8_t *out, const uint8_t *in); | ||||
} | } | ||||
namespace sha256d64_shani { | |||||
void Transform_2way(unsigned char *out, const unsigned char *in); | |||||
} | |||||
namespace sha256_shani { | |||||
void Transform(uint32_t *s, const unsigned char *chunk, size_t blocks); | |||||
} | |||||
// Internal implementation code. | // Internal implementation code. | ||||
namespace { | namespace { | ||||
/// Internal SHA-256 implementation. | /// Internal SHA-256 implementation. | ||||
namespace sha256 { | namespace sha256 { | ||||
inline uint32_t Ch(uint32_t x, uint32_t y, uint32_t z) { | inline uint32_t Ch(uint32_t x, uint32_t y, uint32_t z) { | ||||
return z ^ (x & (y ^ z)); | return z ^ (x & (y ^ z)); | ||||
} | } | ||||
inline uint32_t Maj(uint32_t x, uint32_t y, uint32_t z) { | inline uint32_t Maj(uint32_t x, uint32_t y, uint32_t z) { | ||||
▲ Show 20 Lines • Show All 565 Lines • ▼ Show 20 Lines | void TransformD64Wrapper(uint8_t *out, const uint8_t *in) { | ||||
WriteBE32(out + 16, s[4]); | WriteBE32(out + 16, s[4]); | ||||
WriteBE32(out + 20, s[5]); | WriteBE32(out + 20, s[5]); | ||||
WriteBE32(out + 24, s[6]); | WriteBE32(out + 24, s[6]); | ||||
WriteBE32(out + 28, s[7]); | WriteBE32(out + 28, s[7]); | ||||
} | } | ||||
TransformType Transform = sha256::Transform; | TransformType Transform = sha256::Transform; | ||||
TransformD64Type TransformD64 = sha256::TransformD64; | TransformD64Type TransformD64 = sha256::TransformD64; | ||||
TransformD64Type TransformD64_2way = nullptr; | |||||
TransformD64Type TransformD64_4way = nullptr; | TransformD64Type TransformD64_4way = nullptr; | ||||
TransformD64Type TransformD64_8way = nullptr; | TransformD64Type TransformD64_8way = nullptr; | ||||
bool SelfTest() { | bool SelfTest() { | ||||
// Input state (equal to the initial SHA256 state) | // Input state (equal to the initial SHA256 state) | ||||
static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, | static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, | ||||
0xa54ff53aul, 0x510e527ful, 0x9b05688cul, | 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, | ||||
0x1f83d9abul, 0x5be0cd19ul}; | 0x1f83d9abul, 0x5be0cd19ul}; | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | bool SelfTest() { | ||||
// Test TransformD64 | // Test TransformD64 | ||||
{ | { | ||||
uint8_t out[32]; | uint8_t out[32]; | ||||
TransformD64(out, data + 1); | TransformD64(out, data + 1); | ||||
if (!std::equal(out, out + 32, result_d64)) return false; | if (!std::equal(out, out + 32, result_d64)) return false; | ||||
} | } | ||||
// Test TransformD64_2way, if available. | |||||
if (TransformD64_2way) { | |||||
unsigned char out[64]; | |||||
TransformD64_2way(out, data + 1); | |||||
if (!std::equal(out, out + 64, result_d64)) return false; | |||||
} | |||||
// Test TransformD64_4way, if available. | // Test TransformD64_4way, if available. | ||||
if (TransformD64_4way) { | if (TransformD64_4way) { | ||||
uint8_t out[128]; | uint8_t out[128]; | ||||
TransformD64_4way(out, data + 1); | TransformD64_4way(out, data + 1); | ||||
if (!std::equal(out, out + 128, result_d64)) return false; | if (!std::equal(out, out + 128, result_d64)) return false; | ||||
} | } | ||||
// Test TransformD64_8way, if available. | // Test TransformD64_8way, if available. | ||||
if (TransformD64_8way) { | if (TransformD64_8way) { | ||||
uint8_t out[256]; | uint8_t out[256]; | ||||
TransformD64_8way(out, data + 1); | TransformD64_8way(out, data + 1); | ||||
if (!std::equal(out, out + 256, result_d64)) return false; | if (!std::equal(out, out + 256, result_d64)) return false; | ||||
} | } | ||||
return true; | return true; | ||||
} | } | ||||
#if defined(USE_ASM) && \ | #if defined(USE_ASM) && \ | ||||
(defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) | (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) | ||||
// We can't use cpuid.h's __get_cpuid as it does not support subleafs. | // We can't use cpuid.h's __get_cpuid as it does not support subleafs. | ||||
void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t &a, uint32_t &b, | inline void cpuid(uint32_t leaf, uint32_t subleaf, uint32_t &a, uint32_t &b, | ||||
uint32_t &c, uint32_t &d) { | uint32_t &c, uint32_t &d) { | ||||
#ifdef __GNUC__ | |||||
__cpuid_count(leaf, subleaf, a, b, c, d); | |||||
#else | |||||
__asm__("cpuid" | __asm__("cpuid" | ||||
: "=a"(a), "=b"(b), "=c"(c), "=d"(d) | : "=a"(a), "=b"(b), "=c"(c), "=d"(d) | ||||
: "0"(leaf), "2"(subleaf)); | : "0"(leaf), "2"(subleaf)); | ||||
#endif | |||||
} | } | ||||
/** Check whether the OS has enabled AVX registers. */ | /** Check whether the OS has enabled AVX registers. */ | ||||
bool AVXEnabled() { | bool AVXEnabled() { | ||||
uint32_t a, d; | uint32_t a, d; | ||||
__asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); | __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); | ||||
return (a & 6) == 6; | return (a & 6) == 6; | ||||
} | } | ||||
#endif | #endif | ||||
} // namespace | } // namespace | ||||
std::string SHA256AutoDetect() { | std::string SHA256AutoDetect() { | ||||
std::string ret = "standard"; | std::string ret = "standard"; | ||||
#if defined(USE_ASM) && \ | #if defined(USE_ASM) && \ | ||||
(defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) | (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) | ||||
// Silence unused warning (in case ENABLE_AVX2 is not defined) | bool have_sse4 = false; | ||||
bool have_xsave = false; | |||||
bool have_avx = false; | |||||
bool have_avx2 = false; | |||||
bool have_shani = false; | |||||
bool enabled_avx = false; | |||||
(void)AVXEnabled; | (void)AVXEnabled; | ||||
(void)have_sse4; | |||||
(void)have_avx; | |||||
(void)have_xsave; | |||||
(void)have_avx2; | |||||
(void)have_shani; | |||||
(void)enabled_avx; | |||||
uint32_t eax, ebx, ecx, edx; | uint32_t eax, ebx, ecx, edx; | ||||
cpuid(1, 0, eax, ebx, ecx, edx); | cpuid(1, 0, eax, ebx, ecx, edx); | ||||
if ((ecx >> 19) & 1) { | have_sse4 = (ecx >> 19) & 1; | ||||
have_xsave = (ecx >> 27) & 1; | |||||
have_avx = (ecx >> 28) & 1; | |||||
if (have_xsave && have_avx) { | |||||
enabled_avx = AVXEnabled(); | |||||
} | |||||
if (have_sse4) { | |||||
cpuid(7, 0, eax, ebx, ecx, edx); | |||||
have_avx2 = (ebx >> 5) & 1; | |||||
have_shani = (ebx >> 29) & 1; | |||||
} | |||||
#if defined(ENABLE_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) | |||||
if (have_shani) { | |||||
Transform = sha256_shani::Transform; | |||||
TransformD64 = TransformD64Wrapper<sha256_shani::Transform>; | |||||
TransformD64_2way = sha256d64_shani::Transform_2way; | |||||
ret = "shani(1way,2way)"; | |||||
have_sse4 = false; // Disable SSE4/AVX2; | |||||
have_avx2 = false; | |||||
} | |||||
#endif | |||||
if (have_sse4) { | |||||
#if defined(__x86_64__) || defined(__amd64__) | #if defined(__x86_64__) || defined(__amd64__) | ||||
Transform = sha256_sse4::Transform; | Transform = sha256_sse4::Transform; | ||||
TransformD64 = TransformD64Wrapper<sha256_sse4::Transform>; | TransformD64 = TransformD64Wrapper<sha256_sse4::Transform>; | ||||
ret = "sse4(1way)"; | |||||
#endif | #endif | ||||
#if defined(ENABLE_SSE41) && !defined(BUILD_BITCOIN_INTERNAL) | #if defined(ENABLE_SSE41) && !defined(BUILD_BITCOIN_INTERNAL) | ||||
TransformD64_4way = sha256d64_sse41::Transform_4way; | TransformD64_4way = sha256d64_sse41::Transform_4way; | ||||
ret = "sse4(1way+4way)"; | ret += ",sse41(4way)"; | ||||
#endif | |||||
} | |||||
#if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL) | #if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL) | ||||
if (((ecx >> 27) & 1) && ((ecx >> 28) & 1)) { // XSAVE and AVX | if (have_avx2 && have_avx && enabled_avx) { | ||||
cpuid(7, 0, eax, ebx, ecx, edx); | |||||
if ((ebx >> 5) & 1) { // AVX2 flag | |||||
if (AVXEnabled()) { // OS has enabled AVX registers | |||||
TransformD64_8way = sha256d64_avx2::Transform_8way; | TransformD64_8way = sha256d64_avx2::Transform_8way; | ||||
ret += ",avx2(8way)"; | ret += ",avx2(8way)"; | ||||
} | } | ||||
} | |||||
} | |||||
#endif | #endif | ||||
#else | |||||
ret = "sse4"; | |||||
#endif | |||||
} | |||||
#endif | #endif | ||||
assert(SelfTest()); | assert(SelfTest()); | ||||
return ret; | return ret; | ||||
} | } | ||||
////// SHA-256 | ////// SHA-256 | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | void SHA256D64(uint8_t *out, const uint8_t *in, size_t blocks) { | ||||
if (TransformD64_4way) { | if (TransformD64_4way) { | ||||
while (blocks >= 4) { | while (blocks >= 4) { | ||||
TransformD64_4way(out, in); | TransformD64_4way(out, in); | ||||
out += 128; | out += 128; | ||||
in += 256; | in += 256; | ||||
blocks -= 4; | blocks -= 4; | ||||
} | } | ||||
} | } | ||||
if (TransformD64_2way) { | |||||
while (blocks >= 2) { | |||||
TransformD64_2way(out, in); | |||||
out += 64; | |||||
in += 128; | |||||
blocks -= 2; | |||||
} | |||||
} | |||||
while (blocks) { | while (blocks) { | ||||
TransformD64(out, in); | TransformD64(out, in); | ||||
out += 32; | out += 32; | ||||
in += 64; | in += 64; | ||||
--blocks; | --blocks; | ||||
} | } | ||||
} | } |