7#define _mm_shuffle_ps2(a, b, c) \
9 _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
12 return _mm_loadu_si128((
const __m128i *)src);
16 _mm_storeu_si128((__m128i *)dest, src);
19INLINE __m128i
addv(__m128i a, __m128i b) {
return _mm_add_epi32(a, b); }
22INLINE __m128i
xorv(__m128i a, __m128i b) {
return _mm_xor_si128(a, b); }
27 return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
31 return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1);
35 return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
39 return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8));
43 return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
46INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
49 *row3 =
xorv(*row3, *row0);
51 *row2 =
addv(*row2, *row3);
52 *row1 =
xorv(*row1, *row2);
56INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
59 *row3 =
xorv(*row3, *row0);
61 *row2 =
addv(*row2, *row3);
62 *row1 =
xorv(*row1, *row2);
70 *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
71 *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
72 *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
76 *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
77 *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
78 *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
82 const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
83 __m128i mask = _mm_set1_epi16(imm8);
84 mask = _mm_and_si128(mask, bits);
85 mask = _mm_cmpeq_epi16(mask, bits);
86 return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a));
98 __m128i m0 =
loadu(&
block[
sizeof(__m128i) * 0]);
99 __m128i m1 =
loadu(&
block[
sizeof(__m128i) * 1]);
100 __m128i m2 =
loadu(&
block[
sizeof(__m128i) * 2]);
101 __m128i m3 =
loadu(&
block[
sizeof(__m128i) * 3]);
103 __m128i t0, t1, t2, t3, tt;
108 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
110 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
113 t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3));
114 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
116 t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3));
117 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
127 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
128 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
130 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
132 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
134 t2 = _mm_unpacklo_epi64(m3, m1);
136 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
137 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
138 t3 = _mm_unpackhi_epi32(m1, m3);
139 tt = _mm_unpacklo_epi32(m2, t3);
140 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
141 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
150 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
151 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
153 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
155 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
157 t2 = _mm_unpacklo_epi64(m3, m1);
159 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
160 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
161 t3 = _mm_unpackhi_epi32(m1, m3);
162 tt = _mm_unpacklo_epi32(m2, t3);
163 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
164 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
173 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
174 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
176 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
178 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
180 t2 = _mm_unpacklo_epi64(m3, m1);
182 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
183 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
184 t3 = _mm_unpackhi_epi32(m1, m3);
185 tt = _mm_unpacklo_epi32(m2, t3);
186 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
187 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
196 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
197 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
199 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
201 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
203 t2 = _mm_unpacklo_epi64(m3, m1);
205 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
206 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
207 t3 = _mm_unpackhi_epi32(m1, m3);
208 tt = _mm_unpacklo_epi32(m2, t3);
209 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
210 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
219 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
220 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
222 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
224 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
226 t2 = _mm_unpacklo_epi64(m3, m1);
228 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
229 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
230 t3 = _mm_unpackhi_epi32(m1, m3);
231 tt = _mm_unpacklo_epi32(m2, t3);
232 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
233 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
242 t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
243 g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
245 tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
247 g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
249 t2 = _mm_unpacklo_epi64(m3, m1);
251 t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
252 g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
253 t3 = _mm_unpackhi_epi32(m1, m3);
254 tt = _mm_unpacklo_epi32(m2, t3);
255 t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
256 g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
287 v[0] =
addv(v[0], v[4]);
288 v[1] =
addv(v[1], v[5]);
289 v[2] =
addv(v[2], v[6]);
290 v[3] =
addv(v[3], v[7]);
291 v[12] =
xorv(v[12], v[0]);
292 v[13] =
xorv(v[13], v[1]);
293 v[14] =
xorv(v[14], v[2]);
294 v[15] =
xorv(v[15], v[3]);
295 v[12] =
rot16(v[12]);
296 v[13] =
rot16(v[13]);
297 v[14] =
rot16(v[14]);
298 v[15] =
rot16(v[15]);
299 v[8] =
addv(v[8], v[12]);
300 v[9] =
addv(v[9], v[13]);
301 v[10] =
addv(v[10], v[14]);
302 v[11] =
addv(v[11], v[15]);
303 v[4] =
xorv(v[4], v[8]);
304 v[5] =
xorv(v[5], v[9]);
305 v[6] =
xorv(v[6], v[10]);
306 v[7] =
xorv(v[7], v[11]);
315 v[0] =
addv(v[0], v[4]);
316 v[1] =
addv(v[1], v[5]);
317 v[2] =
addv(v[2], v[6]);
318 v[3] =
addv(v[3], v[7]);
319 v[12] =
xorv(v[12], v[0]);
320 v[13] =
xorv(v[13], v[1]);
321 v[14] =
xorv(v[14], v[2]);
322 v[15] =
xorv(v[15], v[3]);
327 v[8] =
addv(v[8], v[12]);
328 v[9] =
addv(v[9], v[13]);
329 v[10] =
addv(v[10], v[14]);
330 v[11] =
addv(v[11], v[15]);
331 v[4] =
xorv(v[4], v[8]);
332 v[5] =
xorv(v[5], v[9]);
333 v[6] =
xorv(v[6], v[10]);
334 v[7] =
xorv(v[7], v[11]);
344 v[0] =
addv(v[0], v[5]);
345 v[1] =
addv(v[1], v[6]);
346 v[2] =
addv(v[2], v[7]);
347 v[3] =
addv(v[3], v[4]);
348 v[15] =
xorv(v[15], v[0]);
349 v[12] =
xorv(v[12], v[1]);
350 v[13] =
xorv(v[13], v[2]);
351 v[14] =
xorv(v[14], v[3]);
352 v[15] =
rot16(v[15]);
353 v[12] =
rot16(v[12]);
354 v[13] =
rot16(v[13]);
355 v[14] =
rot16(v[14]);
356 v[10] =
addv(v[10], v[15]);
357 v[11] =
addv(v[11], v[12]);
358 v[8] =
addv(v[8], v[13]);
359 v[9] =
addv(v[9], v[14]);
360 v[5] =
xorv(v[5], v[10]);
361 v[6] =
xorv(v[6], v[11]);
362 v[7] =
xorv(v[7], v[8]);
363 v[4] =
xorv(v[4], v[9]);
372 v[0] =
addv(v[0], v[5]);
373 v[1] =
addv(v[1], v[6]);
374 v[2] =
addv(v[2], v[7]);
375 v[3] =
addv(v[3], v[4]);
376 v[15] =
xorv(v[15], v[0]);
377 v[12] =
xorv(v[12], v[1]);
378 v[13] =
xorv(v[13], v[2]);
379 v[14] =
xorv(v[14], v[3]);
384 v[10] =
addv(v[10], v[15]);
385 v[11] =
addv(v[11], v[12]);
386 v[8] =
addv(v[8], v[13]);
387 v[9] =
addv(v[9], v[14]);
388 v[5] =
xorv(v[5], v[10]);
389 v[6] =
xorv(v[6], v[11]);
390 v[7] =
xorv(v[7], v[8]);
391 v[4] =
xorv(v[4], v[9]);
402 __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
403 __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
404 __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
405 __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
408 __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
409 __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
410 __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
411 __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
420 size_t block_offset, __m128i out[16]) {
421 out[0] =
loadu(&inputs[0][block_offset + 0 *
sizeof(__m128i)]);
422 out[1] =
loadu(&inputs[1][block_offset + 0 *
sizeof(__m128i)]);
423 out[2] =
loadu(&inputs[2][block_offset + 0 *
sizeof(__m128i)]);
424 out[3] =
loadu(&inputs[3][block_offset + 0 *
sizeof(__m128i)]);
425 out[4] =
loadu(&inputs[0][block_offset + 1 *
sizeof(__m128i)]);
426 out[5] =
loadu(&inputs[1][block_offset + 1 *
sizeof(__m128i)]);
427 out[6] =
loadu(&inputs[2][block_offset + 1 *
sizeof(__m128i)]);
428 out[7] =
loadu(&inputs[3][block_offset + 1 *
sizeof(__m128i)]);
429 out[8] =
loadu(&inputs[0][block_offset + 2 *
sizeof(__m128i)]);
430 out[9] =
loadu(&inputs[1][block_offset + 2 *
sizeof(__m128i)]);
431 out[10] =
loadu(&inputs[2][block_offset + 2 *
sizeof(__m128i)]);
432 out[11] =
loadu(&inputs[3][block_offset + 2 *
sizeof(__m128i)]);
433 out[12] =
loadu(&inputs[0][block_offset + 3 *
sizeof(__m128i)]);
434 out[13] =
loadu(&inputs[1][block_offset + 3 *
sizeof(__m128i)]);
435 out[14] =
loadu(&inputs[2][block_offset + 3 *
sizeof(__m128i)]);
436 out[15] =
loadu(&inputs[3][block_offset + 3 *
sizeof(__m128i)]);
437 for (
size_t i = 0; i < 4; ++i) {
438 _mm_prefetch((
const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
447 __m128i *out_lo, __m128i *out_hi) {
448 const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter);
449 const __m128i add0 = _mm_set_epi32(3, 2, 1, 0);
450 const __m128i add1 = _mm_and_si128(mask, add0);
451 __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
452 __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)),
453 _mm_xor_si128( l, _mm_set1_epi32(0x80000000)));
454 __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
462 bool increment_counter,
uint8_t flags,
464 __m128i h_vecs[8] = {
468 __m128i counter_low_vec, counter_high_vec;
471 uint8_t block_flags = flags | flags_start;
475 block_flags |= flags_end;
478 __m128i block_flags_vec =
set1(block_flags);
479 __m128i msg_vecs[16];
483 h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
484 h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
486 counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
495 h_vecs[0] =
xorv(v[0], v[8]);
496 h_vecs[1] =
xorv(v[1], v[9]);
497 h_vecs[2] =
xorv(v[2], v[10]);
498 h_vecs[3] =
xorv(v[3], v[11]);
499 h_vecs[4] =
xorv(v[4], v[12]);
500 h_vecs[5] =
xorv(v[5], v[13]);
501 h_vecs[6] =
xorv(v[6], v[14]);
502 h_vecs[7] =
xorv(v[7], v[15]);
511 storeu(h_vecs[0], &out[0 *
sizeof(__m128i)]);
512 storeu(h_vecs[4], &out[1 *
sizeof(__m128i)]);
513 storeu(h_vecs[1], &out[2 *
sizeof(__m128i)]);
514 storeu(h_vecs[5], &out[3 *
sizeof(__m128i)]);
515 storeu(h_vecs[2], &out[4 *
sizeof(__m128i)]);
516 storeu(h_vecs[6], &out[5 *
sizeof(__m128i)]);
517 storeu(h_vecs[3], &out[6 *
sizeof(__m128i)]);
518 storeu(h_vecs[7], &out[7 *
sizeof(__m128i)]);
527 uint8_t block_flags = flags | flags_start;
530 block_flags |= flags_end;
543 uint64_t counter,
bool increment_counter,
546 while (num_inputs >=
DEGREE) {
548 flags_start, flags_end, out);
549 if (increment_counter) {
556 while (num_inputs > 0) {
559 if (increment_counter) {
bbsections Prepares for basic block by splitting functions into clusters of basic blocks
unify loop Fixup each natural loop to have a single exit block
static const uint8_t MSG_SCHEDULE[7][16]
static const uint32_t IV[8]
INLINE uint32_t counter_high(uint64_t counter)
INLINE uint32_t counter_low(uint64_t counter)
INLINE __m128i rot12(__m128i x)
INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
#define _mm_shuffle_ps2(a, b, c)
INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, const uint32_t key[8], uint64_t counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN])
INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
INLINE __m128i rot7(__m128i x)
INLINE void storeu(__m128i src, uint8_t dest[16])
INLINE void transpose_msg_vecs(const uint8_t *const *inputs, size_t block_offset, __m128i out[16])
INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8)
INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3)
INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r)
INLINE __m128i xorv(__m128i a, __m128i b)
INLINE void transpose_vecs(__m128i vecs[DEGREE])
INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3)
INLINE void load_counters(uint64_t counter, bool increment_counter, __m128i *out_lo, __m128i *out_hi)
INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags)
INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, __m128i m)
INLINE __m128i rot16(__m128i x)
INLINE __m128i addv(__m128i a, __m128i b)
static void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out)
INLINE __m128i loadu(const uint8_t src[16])
INLINE __m128i set1(uint32_t x)
INLINE __m128i rot8(__m128i x)
#define blake3_compress_xof_sse2
#define blake3_compress_in_place_sse2
#define blake3_hash_many_sse2