Changeset View
Changeset View
Standalone View
Standalone View
src/crypto/sha256_shani.cpp
Show All 9 Lines | |||||
#include <cstdint> | #include <cstdint> | ||||
#include <immintrin.h> | #include <immintrin.h> | ||||
#include <crypto/common.h> | #include <crypto/common.h> | ||||
namespace { | namespace { | ||||
const __m128i MASK = | alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, | ||||
_mm_set_epi64x(0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL); | 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, | ||||
const __m128i INIT0 = | 0x0f, 0x0e, 0x0d, 0x0c}; | ||||
_mm_set_epi64x(0x6a09e667bb67ae85ull, 0x510e527f9b05688cull); | alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, | ||||
const __m128i INIT1 = | 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, | ||||
_mm_set_epi64x(0x3c6ef372a54ff53aull, 0x1f83d9ab5be0cd19ull); | 0x67, 0xe6, 0x09, 0x6a}; | ||||
alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, | |||||
0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, | |||||
0x72, 0xf3, 0x6e, 0x3c}; | |||||
inline void __attribute__((always_inline)) | inline void __attribute__((always_inline)) | ||||
QuadRound(__m128i &state0, __m128i &state1, uint64_t k1, uint64_t k0) { | QuadRound(__m128i &state0, __m128i &state1, uint64_t k1, uint64_t k0) { | ||||
const __m128i msg = _mm_set_epi64x(k1, k0); | const __m128i msg = _mm_set_epi64x(k1, k0); | ||||
state1 = _mm_sha256rnds2_epu32(state1, state0, msg); | state1 = _mm_sha256rnds2_epu32(state1, state0, msg); | ||||
state0 = | state0 = | ||||
_mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); | _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); | ||||
} | } | ||||
Show All 34 Lines | |||||
inline void __attribute__((always_inline)) Unshuffle(__m128i &s0, __m128i &s1) { | inline void __attribute__((always_inline)) Unshuffle(__m128i &s0, __m128i &s1) { | ||||
const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B); | const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B); | ||||
const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1); | const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1); | ||||
s0 = _mm_blend_epi16(t1, t2, 0xF0); | s0 = _mm_blend_epi16(t1, t2, 0xF0); | ||||
s1 = _mm_alignr_epi8(t2, t1, 0x08); | s1 = _mm_alignr_epi8(t2, t1, 0x08); | ||||
} | } | ||||
__m128i inline __attribute__((always_inline)) Load(const uint8_t *in) { | __m128i inline __attribute__((always_inline)) Load(const uint8_t *in) { | ||||
return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)in), MASK); | return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)in), | ||||
_mm_load_si128((const __m128i *)MASK)); | |||||
} | } | ||||
inline void __attribute__((always_inline)) Save(uint8_t *out, __m128i s) { | inline void __attribute__((always_inline)) Save(uint8_t *out, __m128i s) { | ||||
_mm_storeu_si128((__m128i *)out, _mm_shuffle_epi8(s, MASK)); | _mm_storeu_si128( | ||||
(__m128i *)out, | |||||
_mm_shuffle_epi8(s, _mm_load_si128((const __m128i *)MASK))); | |||||
} | } | ||||
} // namespace | } // namespace | ||||
namespace sha256_shani { | namespace sha256_shani { | ||||
void Transform(uint32_t *s, const uint8_t *chunk, size_t blocks) { | void Transform(uint32_t *s, const uint8_t *chunk, size_t blocks) { | ||||
__m128i m0, m1, m2, m3, s0, s1, so0, so1; | __m128i m0, m1, m2, m3, s0, s1, so0, so1; | ||||
/* Load state */ | /* Load state */ | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | |||||
namespace sha256d64_shani { | namespace sha256d64_shani { | ||||
void Transform_2way(uint8_t *out, const uint8_t *in) { | void Transform_2way(uint8_t *out, const uint8_t *in) { | ||||
__m128i am0, am1, am2, am3, as0, as1, aso0, aso1; | __m128i am0, am1, am2, am3, as0, as1, aso0, aso1; | ||||
__m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1; | __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1; | ||||
/* Transform 1 */ | /* Transform 1 */ | ||||
bs0 = as0 = INIT0; | bs0 = as0 = _mm_load_si128((const __m128i *)INIT0); | ||||
bs1 = as1 = INIT1; | bs1 = as1 = _mm_load_si128((const __m128i *)INIT1); | ||||
am0 = Load(in); | am0 = Load(in); | ||||
bm0 = Load(in + 64); | bm0 = Load(in + 64); | ||||
QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); | QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); | ||||
QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); | QuadRound(bs0, bs1, bm0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); | ||||
am1 = Load(in + 16); | am1 = Load(in + 16); | ||||
bm1 = Load(in + 80); | bm1 = Load(in + 80); | ||||
QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | ||||
QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | void Transform_2way(uint8_t *out, const uint8_t *in) { | ||||
ShiftMessageC(am0, am1, am2); | ShiftMessageC(am0, am1, am2); | ||||
ShiftMessageC(bm0, bm1, bm2); | ShiftMessageC(bm0, bm1, bm2); | ||||
QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | ||||
QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | ||||
ShiftMessageC(am1, am2, am3); | ShiftMessageC(am1, am2, am3); | ||||
ShiftMessageC(bm1, bm2, bm3); | ShiftMessageC(bm1, bm2, bm3); | ||||
QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); | QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); | ||||
QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); | QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); | ||||
as0 = _mm_add_epi32(as0, INIT0); | as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i *)INIT0)); | ||||
bs0 = _mm_add_epi32(bs0, INIT0); | bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i *)INIT0)); | ||||
as1 = _mm_add_epi32(as1, INIT1); | as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i *)INIT1)); | ||||
bs1 = _mm_add_epi32(bs1, INIT1); | bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i *)INIT1)); | ||||
/* Transform 2 */ | /* Transform 2 */ | ||||
aso0 = as0; | aso0 = as0; | ||||
bso0 = bs0; | bso0 = bs0; | ||||
aso1 = as1; | aso1 = as1; | ||||
bso1 = bs1; | bso1 = bs1; | ||||
QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); | QuadRound(as0, as1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); | ||||
QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); | QuadRound(bs0, bs1, 0xe9b5dba5b5c0fbcfull, 0x71374491c28a2f98ull); | ||||
Show All 36 Lines | void Transform_2way(uint8_t *out, const uint8_t *in) { | ||||
Unshuffle(as0, as1); | Unshuffle(as0, as1); | ||||
Unshuffle(bs0, bs1); | Unshuffle(bs0, bs1); | ||||
am0 = as0; | am0 = as0; | ||||
bm0 = bs0; | bm0 = bs0; | ||||
am1 = as1; | am1 = as1; | ||||
bm1 = bs1; | bm1 = bs1; | ||||
/* Transform 3 */ | /* Transform 3 */ | ||||
bs0 = as0 = INIT0; | bs0 = as0 = _mm_load_si128((const __m128i *)INIT0); | ||||
bs1 = as1 = INIT1; | bs1 = as1 = _mm_load_si128((const __m128i *)INIT1); | ||||
QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); | QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); | ||||
QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); | QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); | ||||
QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | ||||
QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | QuadRound(bs0, bs1, bm1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); | ||||
ShiftMessageA(am0, am1); | ShiftMessageA(am0, am1); | ||||
ShiftMessageA(bm0, bm1); | ShiftMessageA(bm0, bm1); | ||||
bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull); | bm2 = am2 = _mm_set_epi64x(0x0ull, 0x80000000ull); | ||||
QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull); | QuadRound(as0, as1, 0x550c7dc3243185beull, 0x12835b015807aa98ull); | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | void Transform_2way(uint8_t *out, const uint8_t *in) { | ||||
ShiftMessageC(am0, am1, am2); | ShiftMessageC(am0, am1, am2); | ||||
ShiftMessageC(bm0, bm1, bm2); | ShiftMessageC(bm0, bm1, bm2); | ||||
QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | QuadRound(as0, as1, am2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | ||||
QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | QuadRound(bs0, bs1, bm2, 0x8cc7020884c87814ull, 0x78a5636f748f82eeull); | ||||
ShiftMessageC(am1, am2, am3); | ShiftMessageC(am1, am2, am3); | ||||
ShiftMessageC(bm1, bm2, bm3); | ShiftMessageC(bm1, bm2, bm3); | ||||
QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); | QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); | ||||
QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); | QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); | ||||
as0 = _mm_add_epi32(as0, INIT0); | as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i *)INIT0)); | ||||
bs0 = _mm_add_epi32(bs0, INIT0); | bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i *)INIT0)); | ||||
as1 = _mm_add_epi32(as1, INIT1); | as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i *)INIT1)); | ||||
bs1 = _mm_add_epi32(bs1, INIT1); | bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i *)INIT1)); | ||||
/* Extract hash into out */ | /* Extract hash into out */ | ||||
Unshuffle(as0, as1); | Unshuffle(as0, as1); | ||||
Unshuffle(bs0, bs1); | Unshuffle(bs0, bs1); | ||||
Save(out, as0); | Save(out, as0); | ||||
Save(out + 16, as1); | Save(out + 16, as1); | ||||
Save(out + 32, bs0); | Save(out + 32, bs0); | ||||
Save(out + 48, bs1); | Save(out + 48, bs1); | ||||
} | } | ||||
} // namespace sha256d64_shani | } // namespace sha256d64_shani | ||||
#endif | #endif |