8 #if CRYPTOPP_MSC_VERSION 9 # pragma warning(disable: 4189) 12 #ifndef CRYPTOPP_IMPORTS 13 #ifndef CRYPTOPP_GENERATE_X64_MASM 16 #if (defined(CRYPTOPP_LLVM_CLANG_VERSION) && (CRYPTOPP_LLVM_CLANG_VERSION < 30400)) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER) || (defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5140) 17 # undef CRYPTOPP_X86_ASM_AVAILABLE 18 # undef CRYPTOPP_X32_ASM_AVAILABLE 19 # undef CRYPTOPP_X64_ASM_AVAILABLE 20 # undef CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 21 # undef CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 22 # undef CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 23 # define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0 24 # define CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 0 25 # define CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 0 33 word16
GCM_Base::s_reductionTable[256];
34 volatile
bool GCM_Base::s_reductionTableInitialized = false;
36 void GCM_Base::GCTR::IncrementCounterBy256()
43 void gcm_gf_mult(
const unsigned char *a,
const unsigned char *b,
unsigned char *c)
45 word64 Z0=0, Z1=0, V0, V1;
48 Block::Get(a)(V0)(V1);
50 for (
int i=0; i<16; i++)
52 for (
int j=0x80; j!=0; j>>=1)
58 V1 = (V1>>1) | (V0<<63);
59 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
65 __m128i _mm_clmulepi64_si128(
const __m128i &a,
const __m128i &b,
int i)
75 for (
int i=0; i<16; i++)
76 ((byte *)&output)[i] = c.
GetByte(i);
81 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 82 inline static void SSE2_Xor16(byte *a,
const byte *b,
const byte *c)
84 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 88 *(__m128i *)(
void *)a = _mm_xor_si128(*(__m128i *)(
void *)b, *(__m128i *)(
void *)c);
90 asm (
"movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" :
"=m" (a[0]) :
"m"(b[0]),
"m"(c[0]));
95 #if CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE 96 inline static void NEON_Xor16(byte *a,
const byte *b,
const byte *c)
98 assert(
IsAlignedOn(a,GetAlignmentOf<uint64x2_t>()));
99 assert(
IsAlignedOn(b,GetAlignmentOf<uint64x2_t>()));
100 assert(
IsAlignedOn(c,GetAlignmentOf<uint64x2_t>()));
101 *(uint64x2_t*)a = veorq_u64(*(uint64x2_t*)b, *(uint64x2_t*)c);
105 inline static void Xor16(byte *a,
const byte *b,
const byte *c)
110 ((word64 *)(
void *)a)[0] = ((word64 *)(
void *)b)[0] ^ ((word64 *)(
void *)c)[0];
111 ((word64 *)(
void *)a)[1] = ((word64 *)(
void *)b)[1] ^ ((word64 *)(
void *)c)[1];
114 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 115 CRYPTOPP_ALIGN_DATA(16)
116 static const word64 s_clmulConstants64[] = {
117 W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
118 W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
119 W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
121 static const __m128i *s_clmulConstants = (
const __m128i *)(
const void *)s_clmulConstants64;
122 static const unsigned int s_clmulTableSizeInBlocks = 8;
124 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2,
const __m128i &r)
138 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301 139 c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
141 c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
143 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
144 c0 = _mm_srli_si128(c0, 8);
145 c0 = _mm_xor_si128(c0, c1);
146 c0 = _mm_slli_epi64(c0, 1);
147 c0 = _mm_clmulepi64_si128(c0, r, 0);
148 c2 = _mm_xor_si128(c2, c0);
149 c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
150 c1 = _mm_unpacklo_epi64(c1, c2);
151 c1 = _mm_srli_epi64(c1, 63);
152 c2 = _mm_slli_epi64(c2, 1);
153 return _mm_xor_si128(c2, c1);
156 inline __m128i CLMUL_GF_Mul(
const __m128i &x,
const __m128i &h,
const __m128i &r)
158 const __m128i c0 = _mm_clmulepi64_si128(x,h,0);
159 const __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
160 const __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
162 return CLMUL_Reduce(c0, c1, c2, r);
166 #if CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE 168 CRYPTOPP_ALIGN_DATA(16)
169 static const word64 s_clmulConstants64[] = {
170 W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
171 W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
172 W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)
175 static const uint64x2_t *s_clmulConstants = (
const uint64x2_t *)s_clmulConstants64;
176 static const unsigned int s_clmulTableSizeInBlocks = 8;
178 inline uint64x2_t PMULL_Reduce(uint64x2_t c0, uint64x2_t c1, uint64x2_t c2,
const uint64x2_t &r)
182 c1 = veorq_u64(c1, (uint64x2_t)vextq_u8(vdupq_n_u8(0), (uint8x16_t)c0, 8));
183 c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(c0, 0), vgetq_lane_u64(r, 1)));
184 c0 = (uint64x2_t)vextq_u8((uint8x16_t)c0, vdupq_n_u8(0), 8);
185 c0 = veorq_u64(c0, c1);
186 c0 = vshlq_n_u64(c0, 1);
187 c0 = (uint64x2_t)vmull_p64(vgetq_lane_u64(c0, 0), vgetq_lane_u64(r, 0));
188 c2 = veorq_u64(c2, c0);
189 c2 = veorq_u64(c2, (uint64x2_t)vextq_u8((uint8x16_t)c1, vdupq_n_u8(0), 8));
190 c1 = vcombine_u64(vget_low_u64(c1), vget_low_u64(c2));
191 c1 = vshrq_n_u64(c1, 63);
192 c2 = vshlq_n_u64(c2, 1);
194 return veorq_u64(c2, c1);
197 inline uint64x2_t PMULL_GF_Mul(
const uint64x2_t &x,
const uint64x2_t &h,
const uint64x2_t &r)
199 const uint64x2_t c0 = (uint64x2_t)vmull_p64(vgetq_lane_u64(x, 0), vgetq_lane_u64(h, 0));
200 const uint64x2_t c1 = veorq_u64((uint64x2_t)vmull_p64(vgetq_lane_u64(x, 1), vgetq_lane_u64(h,0)),
201 (uint64x2_t)vmull_p64(vgetq_lane_u64(x, 0), vgetq_lane_u64(h, 1)));
202 const uint64x2_t c2 = (uint64x2_t)vmull_high_p64((poly64x2_t)x, (poly64x2_t)h);
204 return PMULL_Reduce(c0, c1, c2, r);
208 void GCM_Base::SetKeyWithoutResync(
const byte *userKey,
size_t keylength,
const NameValuePairs ¶ms)
211 blockCipher.
SetKey(userKey, keylength, params);
213 if (blockCipher.
BlockSize() != REQUIRED_BLOCKSIZE)
216 int tableSize, i, j, k;
218 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 223 tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
226 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE 231 tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
237 tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
239 tableSize = (GetTablesOption() ==
GCM_64K_Tables) ? 64*1024 : 2*1024;
241 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400) 247 m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
248 byte *table = MulTable();
249 byte *hashKey = HashKey();
250 memset(hashKey, 0, REQUIRED_BLOCKSIZE);
253 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 256 const __m128i r = s_clmulConstants[0];
257 __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)(
void *)hashKey), s_clmulConstants[1]);
260 for (i=0; i<tableSize; i+=32)
262 __m128i h1 = CLMUL_GF_Mul(h, h0, r);
263 _mm_storel_epi64((__m128i *)(
void *)(table+i), h);
264 _mm_storeu_si128((__m128i *)(
void *)(table+i+16), h1);
265 _mm_storeu_si128((__m128i *)(
void *)(table+i+8), h);
266 _mm_storel_epi64((__m128i *)(
void *)(table+i+8), h1);
267 h = CLMUL_GF_Mul(h1, h0, r);
272 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE 275 const uint64x2_t r = s_clmulConstants[0];
276 const uint64x2_t t = vld1q_u64((uint64_t *)hashKey);
277 const uint64x2_t h0 = (uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t), vget_low_u64(t)));
280 for (i=0; i<tableSize-32; i+=32)
282 const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
283 vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
284 vst1q_u64((uint64_t *)(table+i+16), h1);
285 vst1q_u64((uint64_t *)(table+i+8), h);
286 vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
287 h = PMULL_GF_Mul(h1, h0, r);
290 const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
291 vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
292 vst1q_u64((uint64_t *)(table+i+16), h1);
293 vst1q_u64((uint64_t *)(table+i+8), h);
294 vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
302 Block::Get(hashKey)(V0)(V1);
304 if (tableSize == 64*1024)
306 for (i=0; i<128; i++)
309 Block::Put(NULL, table+(i/8)*256*16+(
size_t(1)<<(11-k)))(V0)(V1);
312 V1 = (V1>>1) | (V0<<63);
313 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
318 memset(table+i*256*16, 0, 16);
319 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 321 for (j=2; j<=0x80; j*=2)
323 SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
325 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE 327 for (j=2; j<=0x80; j*=2)
329 NEON_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
332 for (j=2; j<=0x80; j*=2)
334 Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
339 if (!s_reductionTableInitialized)
341 s_reductionTable[0] = 0;
344 for (
unsigned int ii=2; ii<=0x80; ii*=2)
348 for (
unsigned int jj=1; jj<ii; jj++)
349 s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
351 s_reductionTableInitialized =
true;
354 for (i=0; i<128-24; i++)
358 Block::Put(NULL, table+1024+(i/32)*256+(
size_t(1)<<(7-k)))(V0)(V1);
360 Block::Put(NULL, table+(i/32)*256+(
size_t(1)<<(11-k)))(V0)(V1);
363 V1 = (V1>>1) | (V0<<63);
364 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
369 memset(table+i*256, 0, 16);
370 memset(table+1024+i*256, 0, 16);
371 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 373 for (j=2; j<=8; j*=2)
376 SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
377 SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
380 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE 382 for (j=2; j<=8; j*=2)
385 NEON_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
386 NEON_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
390 for (j=2; j<=8; j*=2)
393 Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
394 Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
400 inline void GCM_Base::ReverseHashBufferIfNeeded()
402 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 405 __m128i &x = *(__m128i *)(
void *)HashBuffer();
406 x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
408 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE 413 const uint8x16_t x = vrev64q_u8(vld1q_u8(HashBuffer()));
414 vst1q_u8(HashBuffer(), (uint8x16_t)vcombine_u64(vget_high_u64((uint64x2_t)x), vget_low_u64((uint64x2_t)x)));
420 void GCM_Base::Resync(
const byte *iv,
size_t len)
423 byte *hashBuffer = HashBuffer();
427 memcpy(hashBuffer, iv, len);
428 memset(hashBuffer+len, 0, 3);
429 hashBuffer[len+3] = 1;
433 size_t origLen = len;
434 memset(hashBuffer, 0, HASH_BLOCKSIZE);
436 if (len >= HASH_BLOCKSIZE)
438 len = GCM_Base::AuthenticateBlocks(iv, len);
439 iv += (origLen - len);
444 memcpy(m_buffer, iv, len);
445 memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
446 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
450 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
452 ReverseHashBufferIfNeeded();
455 if (m_state >= State_IVSet)
456 m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
458 m_ctr.SetCipherWithIV(cipher, hashBuffer);
460 m_ctr.Seek(HASH_BLOCKSIZE);
462 memset(hashBuffer, 0, HASH_BLOCKSIZE);
468 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) 470 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE 476 #if CRYPTOPP_MSC_VERSION 477 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code 480 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM 482 #ifdef CRYPTOPP_X64_MASM_AVAILABLE 484 void GCM_AuthenticateBlocks_2K(
const byte *data,
size_t blocks, word64 *hashBuffer,
const word16 *reductionTable);
485 void GCM_AuthenticateBlocks_64K(
const byte *data,
size_t blocks, word64 *hashBuffer);
489 #ifndef CRYPTOPP_GENERATE_X64_MASM 491 size_t GCM_Base::AuthenticateBlocks(
const byte *data,
size_t len)
493 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 496 const __m128i *table = (
const __m128i *)(
const void *)MulTable();
497 __m128i x = _mm_load_si128((__m128i *)(
void *)HashBuffer());
498 const __m128i r = s_clmulConstants[0], mask1 = s_clmulConstants[1], mask2 = s_clmulConstants[2];
502 size_t s =
UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
503 __m128i d, d2 = _mm_shuffle_epi8(_mm_loadu_si128((
const __m128i *)(
const void *)(data+(s-1)*16)), mask2);;
504 __m128i c0 = _mm_setzero_si128();
505 __m128i c1 = _mm_setzero_si128();
506 __m128i c2 = _mm_setzero_si128();
510 __m128i h0 = _mm_load_si128(table+i);
511 __m128i h1 = _mm_load_si128(table+i+1);
512 __m128i h2 = _mm_xor_si128(h0, h1);
516 d = _mm_shuffle_epi8(_mm_loadu_si128((
const __m128i *)(
const void *)data), mask1);
517 d = _mm_xor_si128(d, x);
518 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0));
519 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
520 d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
521 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0));
525 d = _mm_shuffle_epi8(_mm_loadu_si128((
const __m128i *)(
const void *)(data+(s-i)*16-8)), mask2);
526 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
527 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
528 d2 = _mm_xor_si128(d2, d);
529 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h2, 1));
533 d = _mm_shuffle_epi8(_mm_loadu_si128((
const __m128i *)(
const void *)data), mask1);
534 d = _mm_xor_si128(d, x);
535 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
536 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 0x11));
537 d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
538 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0x10));
542 d2 = _mm_shuffle_epi8(_mm_loadu_si128((
const __m128i *)(
const void *)(data+(s-i)*16-8)), mask1);
543 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
544 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
545 d = _mm_xor_si128(d, d2);
546 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0x10));
551 c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
552 x = CLMUL_Reduce(c0, c1, c2, r);
555 _mm_store_si128((__m128i *)(
void *)HashBuffer(), x);
558 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE 561 const uint64x2_t *table = (
const uint64x2_t *)MulTable();
562 uint64x2_t x = vld1q_u64((
const uint64_t*)HashBuffer());
563 const uint64x2_t r = s_clmulConstants[0];
567 size_t s =
UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
568 uint64x2_t d, d2 = (uint64x2_t)vrev64q_u8((uint8x16_t)vld1q_u64((
const uint64_t *)(data+(s-1)*16)));
569 uint64x2_t c0 = vdupq_n_u64(0);
570 uint64x2_t c1 = vdupq_n_u64(0);
571 uint64x2_t c2 = vdupq_n_u64(0);
575 const uint64x2_t h0 = vld1q_u64((
const uint64_t*)(table+i));
576 const uint64x2_t h1 = vld1q_u64((
const uint64_t*)(table+i+1));
577 const uint64x2_t h2 = veorq_u64(h0, h1);
581 const uint64x2_t t1 = vld1q_u64((
const uint64_t *)data);
582 d = veorq_u64((uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t1), vget_low_u64(t1))), x);
583 c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 0)));
584 c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 1), vgetq_lane_u64(h1, 0)));
585 d = veorq_u64(d, (uint64x2_t)vcombine_u32(vget_high_u32((uint32x4_t)d), vget_low_u32((uint32x4_t)d)));
586 c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 0)));
591 d = (uint64x2_t)vrev64q_u8((uint8x16_t)vld1q_u64((
const uint64_t *)(data+(s-i)*16-8)));
592 c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 1), vgetq_lane_u64(h0, 0)));
593 c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 1), vgetq_lane_u64(h1, 0)));
594 d2 = veorq_u64(d2, d);
595 c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 1), vgetq_lane_u64(h2, 0)));
600 const uint64x2_t t2 = vld1q_u64((
const uint64_t *)data);
601 d = veorq_u64((uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t2), vget_low_u64(t2))), x);
602 c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 1)));
603 c2 = veorq_u64(c2, (uint64x2_t)vmull_high_p64((poly64x2_t)d, (poly64x2_t)h1));
604 d = veorq_u64(d, (uint64x2_t)vcombine_u32(vget_high_u32((uint32x4_t)d), vget_low_u32((uint32x4_t)d)));
605 c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 1)));
610 const uint64x2_t t3 = vld1q_u64((uint64_t *)(data+(s-i)*16-8));
611 d2 = (uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t3), vget_low_u64(t3)));
612 c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 1)));
613 c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 0), vgetq_lane_u64(h1, 1)));
614 d = veorq_u64(d, d2);
615 c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 1)));
620 c1 = veorq_u64(veorq_u64(c1, c0), c2);
621 x = PMULL_Reduce(c0, c1, c2, r);
624 vst1q_u64((uint64_t *)HashBuffer(), x);
630 word64 *hashBuffer = (word64 *)(
void *)HashBuffer();
631 assert(
IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
633 switch (2*(m_buffer.size()>=64*1024)
634 #
if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
643 byte *table = MulTable();
644 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
648 word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
649 Block::Get(data)(y0)(y1);
653 data += HASH_BLOCKSIZE;
654 len -= HASH_BLOCKSIZE;
656 #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(table+(a*1024)+(b*256)+c+d*8) 658 #ifdef IS_LITTLE_ENDIAN 659 #if CRYPTOPP_BOOL_SLOW_WORD64 660 word32 z0 = (word32)x0;
661 word32 z1 = (word32)(x0>>32);
662 word32 z2 = (word32)x1;
663 word32 z3 = (word32)(x1>>32);
664 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e) 666 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e) 668 #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8) 669 #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8; 671 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e) 672 #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff) 673 #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8; 676 #define GF_MUL_32BY128(op, a, b, c) \ 677 a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\ 678 a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\ 679 b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\ 680 b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\ 681 c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\ 682 c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\ 683 d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\ 684 d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\ 686 GF_MUL_32BY128(=, 0, 0, 0)
687 GF_MUL_32BY128(^=, 0, 1, 1)
688 GF_MUL_32BY128(^=, 1, 0, 2)
689 GF_MUL_32BY128(^=, 1, 1, 3)
691 word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
694 r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
697 r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
703 while (len >= HASH_BLOCKSIZE);
705 hashBuffer[0] = x0; hashBuffer[1] = x1;
711 byte *table = MulTable();
712 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
716 word64 y0, y1, a0, a1;
717 Block::Get(data)(y0)(y1);
721 data += HASH_BLOCKSIZE;
722 len -= HASH_BLOCKSIZE;
724 #undef READ_TABLE_WORD64_COMMON 725 #undef READ_TABLE_WORD64 727 #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(table+(a)*256*16+(c)+(d)*8) 729 #ifdef IS_LITTLE_ENDIAN 730 #if CRYPTOPP_BOOL_SLOW_WORD64 731 word32 z0 = (word32)x0;
732 word32 z1 = (word32)(x0>>32);
733 word32 z2 = (word32)x1;
734 word32 z3 = (word32)(x1>>32);
735 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e) 737 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e) 740 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e) 743 #define GF_MUL_8BY128(op, b, c, d) \ 744 a0 op READ_TABLE_WORD64(b, c, d, 0);\ 745 a1 op READ_TABLE_WORD64(b, c, d, 1);\ 747 GF_MUL_8BY128(=, 0, 0, 0)
748 GF_MUL_8BY128(^=, 0, 0, 1)
749 GF_MUL_8BY128(^=, 0, 0, 2)
750 GF_MUL_8BY128(^=, 0, 0, 3)
751 GF_MUL_8BY128(^=, 0, 1, 0)
752 GF_MUL_8BY128(^=, 0, 1, 1)
753 GF_MUL_8BY128(^=, 0, 1, 2)
754 GF_MUL_8BY128(^=, 0, 1, 3)
755 GF_MUL_8BY128(^=, 1, 2, 0)
756 GF_MUL_8BY128(^=, 1, 2, 1)
757 GF_MUL_8BY128(^=, 1, 2, 2)
758 GF_MUL_8BY128(^=, 1, 2, 3)
759 GF_MUL_8BY128(^=, 1, 3, 0)
760 GF_MUL_8BY128(^=, 1, 3, 1)
761 GF_MUL_8BY128(^=, 1, 3, 2)
762 GF_MUL_8BY128(^=, 1, 3, 3)
766 while (len >= HASH_BLOCKSIZE);
768 hashBuffer[0] = x0; hashBuffer[1] = x1;
771 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM 773 #ifdef CRYPTOPP_X64_MASM_AVAILABLE 775 GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
778 GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
782 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 789 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
791 GCM_AuthenticateBlocks_2K PROC FRAME
799 AS2( mov WORD_REG(cx), data )
800 AS2( mov WORD_REG(dx), len )
801 AS2( mov WORD_REG(si), hashBuffer )
802 AS2( shr WORD_REG(dx), 4 )
805 #
if CRYPTOPP_BOOL_X32
814 AS2( mov AS_REG_7, WORD_REG(di))
815 #elif CRYPTOPP_BOOL_X86
816 AS2( lea AS_REG_7, s_reductionTable)
819 AS2( movdqa xmm0, [WORD_REG(si)] )
821 #define MUL_TABLE_0 WORD_REG(si) + 32
822 #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
823 #define RED_TABLE AS_REG_7
826 AS2( movdqu xmm4, [WORD_REG(cx)] )
827 AS2( pxor xmm0, xmm4 )
829 AS2( movd ebx, xmm0 )
830 AS2( mov eax, AS_HEX(f0f0f0f0) )
833 AS2( and ebx, AS_HEX(f0f0f0f0) )
835 AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
837 AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
840 AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
842 AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
844 #define SSE2_MUL_32BITS(i) \
845 AS2( psrldq xmm0, 4 )\
846 AS2( movd eax, xmm0 )\
847 AS2( and eax, AS_HEX(f0f0f0f0) )\
848 AS2( movzx edi, bh )\
849 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
850 AS2( movzx edi, bl )\
851 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
853 AS2( movzx edi, bh )\
854 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
855 AS2( movzx edi, bl )\
856 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
857 AS2( movd ebx, xmm0 )\
859 AS2( and ebx, AS_HEX(f0f0f0f0) )\
860 AS2( movzx edi, ah )\
861 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
862 AS2( movzx edi, al )\
863 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
865 AS2( movzx edi, ah )\
866 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
867 AS2( movzx edi, al )\
868 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
875 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
877 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
880 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
882 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
884 AS2( movdqa xmm0, xmm3 )
885 AS2( pslldq xmm3, 1 )
886 AS2( pxor xmm2, xmm3 )
887 AS2( movdqa xmm1, xmm2 )
888 AS2( pslldq xmm2, 1 )
889 AS2( pxor xmm5, xmm2 )
891 AS2( psrldq xmm0, 15 )
892 #
if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
893 AS2( movd edi, xmm0 )
894 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
895 AS2( mov WORD_REG(di), xmm0 )
897 AS2( movd WORD_REG(di), xmm0 )
899 AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
902 AS2( movdqa xmm0, xmm5 )
903 AS2( pslldq xmm5, 1 )
904 AS2( pxor xmm4, xmm5 )
906 AS2( psrldq xmm1, 15 )
907 #
if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
908 AS2( movd edi, xmm1 )
909 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
910 AS2( mov WORD_REG(di), xmm1 )
912 AS2( movd WORD_REG(di), xmm1 )
914 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
917 AS2( psrldq xmm0, 15 )
918 #
if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
919 AS2( movd edi, xmm0 )
920 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
921 AS2( mov WORD_REG(di), xmm0 )
923 AS2( movd WORD_REG(di), xmm0 )
925 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
927 AS2( movd xmm0, eax )
928 AS2( pxor xmm0, xmm4 )
930 AS2( add WORD_REG(cx), 16 )
931 AS2( sub WORD_REG(dx), 1 )
935 AS2( movdqa [WORD_REG(si)], xmm0 )
937 #
if CRYPTOPP_BOOL_X32
948 :
"c" (data),
"d" (len/16),
"S" (hashBuffer),
"D" (s_reductionTable)
949 :
"memory",
"cc",
"%eax" 950 #
if CRYPTOPP_BOOL_X64
954 #elif defined(CRYPTOPP_GENERATE_X64_MASM) 959 GCM_AuthenticateBlocks_2K ENDP
970 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
972 GCM_AuthenticateBlocks_64K PROC FRAME
978 AS2( mov WORD_REG(cx), data )
979 AS2( mov WORD_REG(dx), len )
980 AS2( mov WORD_REG(si), hashBuffer )
981 AS2( shr WORD_REG(dx), 4 )
984 AS2( movdqa xmm0, [WORD_REG(si)] )
987 #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
990 AS2( movdqu xmm1, [WORD_REG(cx)] )
991 AS2( pxor xmm1, xmm0 )
992 AS2( pxor xmm0, xmm0 )
994 #undef SSE2_MUL_32BITS
995 #define SSE2_MUL_32BITS(i) \
996 AS2( movd eax, xmm1 )\
997 AS2( psrldq xmm1, 4 )\
998 AS2( movzx edi, al )\
999 AS2( add WORD_REG(di), WORD_REG(di) )\
1000 AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
1001 AS2( movzx edi, ah )\
1002 AS2( add WORD_REG(di), WORD_REG(di) )\
1003 AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
1005 AS2( movzx edi, al )\
1006 AS2( add WORD_REG(di), WORD_REG(di) )\
1007 AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
1008 AS2( movzx edi, ah )\
1009 AS2( add WORD_REG(di), WORD_REG(di) )\
1010 AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
1017 AS2( add WORD_REG(cx), 16 )
1018 AS2( sub WORD_REG(dx), 1 )
1022 AS2( movdqa [WORD_REG(si)], xmm0 )
1027 :
"c" (data),
"d" (len/16),
"S" (hashBuffer)
1028 :
"memory",
"cc",
"%edi",
"%eax" 1030 #elif defined(CRYPTOPP_GENERATE_X64_MASM) 1034 GCM_AuthenticateBlocks_64K ENDP
1040 #ifndef CRYPTOPP_GENERATE_X64_MASM 1046 void GCM_Base::AuthenticateLastHeaderBlock()
1048 if (m_bufferedDataLength > 0)
1050 memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
1051 m_bufferedDataLength = 0;
1052 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1056 void GCM_Base::AuthenticateLastConfidentialBlock()
1058 GCM_Base::AuthenticateLastHeaderBlock();
1060 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1063 void GCM_Base::AuthenticateLastFooterBlock(byte *mac,
size_t macSize)
1066 ReverseHashBufferIfNeeded();
1067 m_ctr.ProcessData(mac, HashBuffer(), macSize);
1072 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM GCM block cipher base implementation.
An invalid argument was detected.
virtual void SetKey(const byte *key, size_t length, const NameValuePairs ¶ms=g_nullNameValuePairs)
Sets or reset the key of this object.
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
Library configuration file.
Access a block of memory.
byte order is little-endian
Polynomial with Coefficients in GF(2)
Interface for one direction (encryption or decryption) of a block cipher.
Use a table with 64K entries.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
const char * TableSize()
int, in bytes
bool HasCLMUL()
Determines Carryless Multiply availability.
const T1 UnsignedMin(const T1 &a, const T2 &b)
Safe comparison of values that could be neagtive and incorrectly promoted.
std::string AlgorithmName() const
Provides the name of this algorithm.
const char * BlockSize()
int, in bytes
Functions for CPU features and intrinsics.
bool HasSSE2()
Determines SSE2 availability.
GCM block cipher mode of operation.
ByteOrder GetNativeByteOrder()
Returns NativeByteOrder as an enumerated ByteOrder value.
byte GetByte(size_t n) const
return the n-th byte
Access a block of memory.
Crypto++ library namespace.
bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
Interface for retrieving values given their names.