LLVM 19.0.0git
blake3_impl.h
Go to the documentation of this file.
1#ifndef BLAKE3_IMPL_H
2#define BLAKE3_IMPL_H
3
4#include <assert.h>
5#include <stdbool.h>
6#include <stddef.h>
7#include <stdint.h>
8#include <string.h>
9
10#include "llvm-c/blake3.h"
11// For \p LLVM_LIBRARY_VISIBILITY
13
14#include "llvm_blake3_prefix.h"
15
16// internal flags
18 CHUNK_START = 1 << 0,
19 CHUNK_END = 1 << 1,
20 PARENT = 1 << 2,
21 ROOT = 1 << 3,
22 KEYED_HASH = 1 << 4,
25};
26
27// This C implementation tries to support recent versions of GCC, Clang, and
28// MSVC.
29#if defined(_MSC_VER)
30#define INLINE static __forceinline
31#else
32#define INLINE static inline __attribute__((always_inline))
33#endif
34
35#if defined(__x86_64__) || defined(_M_X64)
36#define IS_X86
37#define IS_X86_64
38#endif
39
40#if defined(__i386__) || defined(_M_IX86)
41#define IS_X86
42#define IS_X86_32
43#endif
44
45#if defined(__aarch64__) || defined(_M_ARM64)
46#define IS_AARCH64
47#endif
48
49#if defined(IS_X86)
50#if defined(_MSC_VER)
51#include <intrin.h>
52#endif
53#include <immintrin.h>
54#endif
55
56#if !defined(BLAKE3_USE_NEON)
57 // If BLAKE3_USE_NEON not manually set, autodetect based on
58 // AArch64ness and endianness.
59 #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)
60 #define BLAKE3_USE_NEON 1
61 #else
62 #define BLAKE3_USE_NEON 0
63 #endif
64#endif
65
66#if defined(IS_X86)
67#define MAX_SIMD_DEGREE 16
68#elif BLAKE3_USE_NEON == 1
69#define MAX_SIMD_DEGREE 4
70#else
71#define MAX_SIMD_DEGREE 1
72#endif
73
74// There are some places where we want a static size that's equal to the
75// MAX_SIMD_DEGREE, but also at least 2.
76#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
77
78static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
79 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
80 0x1F83D9ABUL, 0x5BE0CD19UL};
81
82static const uint8_t MSG_SCHEDULE[7][16] = {
83 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
84 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
85 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
86 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
87 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
88 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
89 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
90};
91
92/* Find index of the highest set bit */
93/* x is assumed to be nonzero. */
94static unsigned int highest_one(uint64_t x) {
95#if defined(__GNUC__) || defined(__clang__)
96 return 63 ^ __builtin_clzll(x);
97#elif defined(_MSC_VER) && defined(IS_X86_64)
98 unsigned long index;
99 _BitScanReverse64(&index, x);
100 return index;
101#elif defined(_MSC_VER) && defined(IS_X86_32)
102 if(x >> 32) {
103 unsigned long index;
104 _BitScanReverse(&index, (unsigned long)(x >> 32));
105 return 32 + index;
106 } else {
107 unsigned long index;
108 _BitScanReverse(&index, (unsigned long)x);
109 return index;
110 }
111#else
112 unsigned int c = 0;
113 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
114 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
115 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
116 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
117 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
118 if(x & 0x0000000000000002ULL) { c += 1; }
119 return c;
120#endif
121}
122
123// Count the number of 1 bits.
124INLINE unsigned int popcnt(uint64_t x) {
125#if defined(__GNUC__) || defined(__clang__)
126 return __builtin_popcountll(x);
127#else
128 unsigned int count = 0;
129 while (x != 0) {
130 count += 1;
131 x &= x - 1;
132 }
133 return count;
134#endif
135}
136
137// Largest power of two less than or equal to x. As a special case, returns 1
138// when x is 0.
140 return 1ULL << highest_one(x | 1);
141}
142
143INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
144
146 return (uint32_t)(counter >> 32);
147}
148
149INLINE uint32_t load32(const void *src) {
150 const uint8_t *p = (const uint8_t *)src;
151 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
152 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
153}
154
155INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
156 uint32_t key_words[8]) {
157 key_words[0] = load32(&key[0 * 4]);
158 key_words[1] = load32(&key[1 * 4]);
159 key_words[2] = load32(&key[2 * 4]);
160 key_words[3] = load32(&key[3 * 4]);
161 key_words[4] = load32(&key[4 * 4]);
162 key_words[5] = load32(&key[5 * 4]);
163 key_words[6] = load32(&key[6 * 4]);
164 key_words[7] = load32(&key[7 * 4]);
165}
166
167INLINE void store32(void *dst, uint32_t w) {
168 uint8_t *p = (uint8_t *)dst;
169 p[0] = (uint8_t)(w >> 0);
170 p[1] = (uint8_t)(w >> 8);
171 p[2] = (uint8_t)(w >> 16);
172 p[3] = (uint8_t)(w >> 24);
173}
174
175INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
176 store32(&bytes_out[0 * 4], cv_words[0]);
177 store32(&bytes_out[1 * 4], cv_words[1]);
178 store32(&bytes_out[2 * 4], cv_words[2]);
179 store32(&bytes_out[3 * 4], cv_words[3]);
180 store32(&bytes_out[4 * 4], cv_words[4]);
181 store32(&bytes_out[5 * 4], cv_words[5]);
182 store32(&bytes_out[6 * 4], cv_words[6]);
183 store32(&bytes_out[7 * 4], cv_words[7]);
184}
185
188 const uint8_t block[BLAKE3_BLOCK_LEN],
189 uint8_t block_len, uint64_t counter,
190 uint8_t flags);
191
193void blake3_compress_xof(const uint32_t cv[8],
194 const uint8_t block[BLAKE3_BLOCK_LEN],
195 uint8_t block_len, uint64_t counter, uint8_t flags,
196 uint8_t out[64]);
197
199void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
200 size_t blocks, const uint32_t key[8], uint64_t counter,
201 bool increment_counter, uint8_t flags,
202 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
203
205size_t blake3_simd_degree(void);
206
207
208// Declarations for implementation-specific functions.
211 const uint8_t block[BLAKE3_BLOCK_LEN],
212 uint8_t block_len, uint64_t counter,
213 uint8_t flags);
214
217 const uint8_t block[BLAKE3_BLOCK_LEN],
218 uint8_t block_len, uint64_t counter,
219 uint8_t flags, uint8_t out[64]);
220
222void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
223 size_t blocks, const uint32_t key[8],
224 uint64_t counter, bool increment_counter,
225 uint8_t flags, uint8_t flags_start,
226 uint8_t flags_end, uint8_t *out);
227
228#if defined(IS_X86)
229#if !defined(BLAKE3_NO_SSE2)
232 const uint8_t block[BLAKE3_BLOCK_LEN],
233 uint8_t block_len, uint64_t counter,
234 uint8_t flags);
236void blake3_compress_xof_sse2(const uint32_t cv[8],
237 const uint8_t block[BLAKE3_BLOCK_LEN],
238 uint8_t block_len, uint64_t counter,
239 uint8_t flags, uint8_t out[64]);
241void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
242 size_t blocks, const uint32_t key[8],
243 uint64_t counter, bool increment_counter,
244 uint8_t flags, uint8_t flags_start,
245 uint8_t flags_end, uint8_t *out);
246#endif
247#if !defined(BLAKE3_NO_SSE41)
250 const uint8_t block[BLAKE3_BLOCK_LEN],
251 uint8_t block_len, uint64_t counter,
252 uint8_t flags);
254void blake3_compress_xof_sse41(const uint32_t cv[8],
255 const uint8_t block[BLAKE3_BLOCK_LEN],
256 uint8_t block_len, uint64_t counter,
257 uint8_t flags, uint8_t out[64]);
259void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
260 size_t blocks, const uint32_t key[8],
261 uint64_t counter, bool increment_counter,
262 uint8_t flags, uint8_t flags_start,
263 uint8_t flags_end, uint8_t *out);
264#endif
265#if !defined(BLAKE3_NO_AVX2)
267void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
268 size_t blocks, const uint32_t key[8],
269 uint64_t counter, bool increment_counter,
270 uint8_t flags, uint8_t flags_start,
271 uint8_t flags_end, uint8_t *out);
272#endif
273#if !defined(BLAKE3_NO_AVX512)
276 const uint8_t block[BLAKE3_BLOCK_LEN],
277 uint8_t block_len, uint64_t counter,
278 uint8_t flags);
279
281void blake3_compress_xof_avx512(const uint32_t cv[8],
282 const uint8_t block[BLAKE3_BLOCK_LEN],
283 uint8_t block_len, uint64_t counter,
284 uint8_t flags, uint8_t out[64]);
285
287void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
288 size_t blocks, const uint32_t key[8],
289 uint64_t counter, bool increment_counter,
290 uint8_t flags, uint8_t flags_start,
291 uint8_t flags_end, uint8_t *out);
292#endif
293#endif
294
295#if BLAKE3_USE_NEON == 1
297void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
298 size_t blocks, const uint32_t key[8],
299 uint64_t counter, bool increment_counter,
300 uint8_t flags, uint8_t flags_start,
301 uint8_t flags_end, uint8_t *out);
302#endif
303
304
305#endif /* BLAKE3_IMPL_H */
bbsections Prepares for basic block by splitting functions into clusters of basic blocks
#define LLVM_LIBRARY_VISIBILITY
Definition: Compiler.h:131
unify loop Fixup each natural loop to have a single exit block
static const uint8_t MSG_SCHEDULE[7][16]
Definition: blake3_impl.h:82
INLINE unsigned int popcnt(uint64_t x)
Definition: blake3_impl.h:124
LLVM_LIBRARY_VISIBILITY size_t blake3_simd_degree(void)
#define INLINE
Definition: blake3_impl.h:32
static const uint32_t IV[8]
Definition: blake3_impl.h:78
static unsigned int highest_one(uint64_t x)
Definition: blake3_impl.h:94
INLINE uint32_t counter_high(uint64_t counter)
Definition: blake3_impl.h:145
INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8])
Definition: blake3_impl.h:155
INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8])
Definition: blake3_impl.h:175
INLINE uint64_t round_down_to_power_of_2(uint64_t x)
Definition: blake3_impl.h:139
INLINE uint32_t load32(const void *src)
Definition: blake3_impl.h:149
INLINE void store32(void *dst, uint32_t w)
Definition: blake3_impl.h:167
blake3_flags
Definition: blake3_impl.h:17
@ CHUNK_START
Definition: blake3_impl.h:18
@ PARENT
Definition: blake3_impl.h:20
@ KEYED_HASH
Definition: blake3_impl.h:22
@ DERIVE_KEY_MATERIAL
Definition: blake3_impl.h:24
@ DERIVE_KEY_CONTEXT
Definition: blake3_impl.h:23
@ ROOT
Definition: blake3_impl.h:21
@ CHUNK_END
Definition: blake3_impl.h:19
INLINE uint32_t counter_low(uint64_t counter)
Definition: blake3_impl.h:143
#define blake3_compress_in_place_sse41
#define blake3_hash_many_neon
#define blake3_hash_many_avx512
#define blake3_hash_many_avx2
#define blake3_compress_xof_sse2
#define BLAKE3_BLOCK_LEN
#define blake3_hash_many_sse41
#define blake3_compress_xof
#define blake3_compress_xof_sse41
#define blake3_hash_many
#define blake3_compress_in_place_sse2
#define blake3_compress_in_place
#define blake3_compress_xof_avx512
#define BLAKE3_KEY_LEN
#define blake3_compress_xof_portable
#define blake3_hash_many_portable
#define blake3_hash_many_sse2
#define blake3_compress_in_place_portable
#define blake3_compress_in_place_avx512