Crypto++  5.6.5
Free C++ class library of cryptographic schemes
gcm.cpp
1 // gcm.cpp - originally written and placed in the public domain by Wei Dai
2 // ARM and Aarch64 added by Jeffrey Walton. The ARM carryless
3 // multiply routines are less efficient because they shadowed x86.
4 // The precomputed key table integration makes it tricky to use the
5 // more efficient ARMv8 implementation of the multiply and reduce.
6 
7 // use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
8 
9 #include "pch.h"
10 #include "config.h"
11 
12 #if CRYPTOPP_MSC_VERSION
13 # pragma warning(disable: 4189)
14 #endif
15 
16 #ifndef CRYPTOPP_IMPORTS
17 #ifndef CRYPTOPP_GENERATE_X64_MASM
18 
19 // Clang 3.3 integrated assembler crash on Linux.
20 #if (defined(CRYPTOPP_LLVM_CLANG_VERSION) && (CRYPTOPP_LLVM_CLANG_VERSION < 30400))
21 # undef CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
22 #endif
23 
24 // SunCC 5.13 and below crash with AES-NI/CLMUL and C++{03|11}. Disable one or the other.
25 // Also see http://github.com/weidai11/cryptopp/issues/226
26 #if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x513)
27 # undef CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
28 #endif
29 
30 #include "gcm.h"
31 #include "cpu.h"
32 
33 NAMESPACE_BEGIN(CryptoPP)
34 
35 #if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
36 // Different assemblers accept different mnemonics: 'movd eax, xmm0' vs 'movd rax, xmm0' vs 'mov eax, xmm0' vs 'mov rax, xmm0'
37 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER)
38 // 'movd eax, xmm0' only. REG_WORD() macro not used.
39 # define USE_MOVD_REG32 1
40 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
41 // 'movd eax, xmm0' or 'movd rax, xmm0'. REG_WORD() macro supplies REG32 or REG64.
42 # define USE_MOVD_REG32_OR_REG64 1
43 #elif defined(__GNUC__) || defined(_MSC_VER)
44 // 'movd eax, xmm0' or 'movd rax, xmm0'. REG_WORD() macro supplies REG32 or REG64.
45 # define USE_MOVD_REG32_OR_REG64 1
46 #else
47 // 'mov eax, xmm0' or 'mov rax, xmm0'. REG_WORD() macro supplies REG32 or REG64.
48 # define USE_MOV_REG32_OR_REG64 1
49 #endif
50 #endif
51 
52 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64) && CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
53 #if defined(__GNUC__)
54 // Schneiders, Hovsmith and O'Rourke used this trick.
55 // It results in much better code generation in production code
56 // by avoiding D-register spills when using vgetq_lane_u64. The
57 // problem does not surface under minimal test cases.
58 inline uint64x2_t PMULL_00(const uint64x2_t a, const uint64x2_t b)
59 {
60  uint64x2_t r;
61  __asm __volatile("pmull %0.1q, %1.1d, %2.1d \n\t"
62  :"=w" (r) : "w" (a), "w" (b) );
63  return r;
64 }
65 
66 inline uint64x2_t PMULL_01(const uint64x2_t a, const uint64x2_t b)
67 {
68  uint64x2_t r;
69  __asm __volatile("pmull %0.1q, %1.1d, %2.1d \n\t"
70  :"=w" (r) : "w" (a), "w" (vget_high_u64(b)) );
71  return r;
72 }
73 
74 inline uint64x2_t PMULL_10(const uint64x2_t a, const uint64x2_t b)
75 {
76  uint64x2_t r;
77  __asm __volatile("pmull %0.1q, %1.1d, %2.1d \n\t"
78  :"=w" (r) : "w" (vget_high_u64(a)), "w" (b) );
79  return r;
80 }
81 
82 inline uint64x2_t PMULL_11(const uint64x2_t a, const uint64x2_t b)
83 {
84  uint64x2_t r;
85  __asm __volatile("pmull2 %0.1q, %1.2d, %2.2d \n\t"
86  :"=w" (r) : "w" (a), "w" (b) );
87  return r;
88 }
89 
90 inline uint64x2_t VEXT_U8(uint64x2_t a, uint64x2_t b, unsigned int c)
91 {
92  uint64x2_t r;
93  __asm __volatile("ext %0.16b, %1.16b, %2.16b, %3 \n\t"
94  :"=w" (r) : "w" (a), "w" (b), "I" (c) );
95  return r;
96 }
97 
98 // https://github.com/weidai11/cryptopp/issues/366
99 template <unsigned int C>
100 inline uint64x2_t VEXT_U8(uint64x2_t a, uint64x2_t b)
101 {
102  uint64x2_t r;
103  __asm __volatile("ext %0.16b, %1.16b, %2.16b, %3 \n\t"
104  :"=w" (r) : "w" (a), "w" (b), "I" (C) );
105  return r;
106 }
107 #endif // GCC and compatibles
108 
109 #if defined(_MSC_VER)
110 inline uint64x2_t PMULL_00(const uint64x2_t a, const uint64x2_t b)
111 {
112  return (uint64x2_t)(vmull_p64(vgetq_lane_u64(vreinterpretq_u64_u8(a),0),
113  vgetq_lane_u64(vreinterpretq_u64_u8(b),0)));
114 }
115 
116 inline uint64x2_t PMULL_01(const uint64x2_t a, const uint64x2_t b)
117 {
118  return (uint64x2_t)(vmull_p64(vgetq_lane_u64(vreinterpretq_u64_u8(a),0),
119  vgetq_lane_u64(vreinterpretq_u64_u8(b),1)));
120 }
121 
122 inline uint64x2_t PMULL_10(const uint64x2_t a, const uint64x2_t b)
123 {
124  return (uint64x2_t)(vmull_p64(vgetq_lane_u64(vreinterpretq_u64_u8(a),1),
125  vgetq_lane_u64(vreinterpretq_u64_u8(b),0)));
126 }
127 
128 inline uint64x2_t PMULL_11(const uint64x2_t a, const uint64x2_t b)
129 {
130  return (uint64x2_t)(vmull_p64(vgetq_lane_u64(vreinterpretq_u64_u8(a),1),
131  vgetq_lane_u64(vreinterpretq_u64_u8(b),1)));
132 }
133 
134 inline uint64x2_t VEXT_U8(uint64x2_t a, uint64x2_t b, unsigned int c)
135 {
136  return (uint64x2_t)vextq_u8(vreinterpretq_u8_u64(a), vreinterpretq_u8_u64(b), c);
137 }
138 
139 // https://github.com/weidai11/cryptopp/issues/366
140 template <unsigned int C>
141 inline uint64x2_t VEXT_U8(uint64x2_t a, uint64x2_t b)
142 {
143  return (uint64x2_t)vextq_u8(vreinterpretq_u8_u64(a), vreinterpretq_u8_u64(b), C);
144 }
145 #endif // Microsoft and compatibles
146 #endif // CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
147 
148 word16 GCM_Base::s_reductionTable[256];
149 volatile bool GCM_Base::s_reductionTableInitialized = false;
150 
151 void GCM_Base::GCTR::IncrementCounterBy256()
152 {
153  IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
154 }
155 
156 #if 0
157 // preserved for testing
158 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
159 {
160  word64 Z0=0, Z1=0, V0, V1;
161 
163  Block::Get(a)(V0)(V1);
164 
165  for (int i=0; i<16; i++)
166  {
167  for (int j=0x80; j!=0; j>>=1)
168  {
169  int x = b[i] & j;
170  Z0 ^= x ? V0 : 0;
171  Z1 ^= x ? V1 : 0;
172  x = (int)V1 & 1;
173  V1 = (V1>>1) | (V0<<63);
174  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
175  }
176  }
177  Block::Put(NULLPTR, c)(Z0)(Z1);
178 }
179 
180 __m128i _mm_clmulepi64_si128(const __m128i &a, const __m128i &b, int i)
181 {
182  word64 A[1] = {ByteReverse(((word64*)&a)[i&1])};
183  word64 B[1] = {ByteReverse(((word64*)&b)[i>>4])};
184 
185  PolynomialMod2 pa((byte *)A, 8);
186  PolynomialMod2 pb((byte *)B, 8);
187  PolynomialMod2 c = pa*pb;
188 
189  __m128i output;
190  for (int i=0; i<16; i++)
191  ((byte *)&output)[i] = c.GetByte(i);
192  return output;
193 }
194 #endif
195 
196 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
197 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
198 {
199 // SunCC 5.14 crash (bewildering since asserts are not in effect in release builds)
200 // Also see http://github.com/weidai11/cryptopp/issues/226 and http://github.com/weidai11/cryptopp/issues/284
201 # if __SUNPRO_CC
202  *(__m128i *)(void *)a = _mm_xor_si128(*(__m128i *)(void *)b, *(__m128i *)(void *)c);
203 # elif CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
204  CRYPTOPP_ASSERT(IsAlignedOn(a,GetAlignmentOf<__m128i>()));
205  CRYPTOPP_ASSERT(IsAlignedOn(b,GetAlignmentOf<__m128i>()));
206  CRYPTOPP_ASSERT(IsAlignedOn(c,GetAlignmentOf<__m128i>()));
207  *(__m128i *)(void *)a = _mm_xor_si128(*(__m128i *)(void *)b, *(__m128i *)(void *)c);
208 # else
209  asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
210 # endif
211 }
212 #endif
213 
214 #if CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
215 inline static void NEON_Xor16(byte *a, const byte *b, const byte *c)
216 {
217  CRYPTOPP_ASSERT(IsAlignedOn(a,GetAlignmentOf<uint64x2_t>()));
218  CRYPTOPP_ASSERT(IsAlignedOn(b,GetAlignmentOf<uint64x2_t>()));
219  CRYPTOPP_ASSERT(IsAlignedOn(c,GetAlignmentOf<uint64x2_t>()));
220  *(uint64x2_t*)a = veorq_u64(*(uint64x2_t*)b, *(uint64x2_t*)c);
221 }
222 #endif
223 
224 inline static void Xor16(byte *a, const byte *b, const byte *c)
225 {
226  CRYPTOPP_ASSERT(IsAlignedOn(a,GetAlignmentOf<word64>()));
227  CRYPTOPP_ASSERT(IsAlignedOn(b,GetAlignmentOf<word64>()));
228  CRYPTOPP_ASSERT(IsAlignedOn(c,GetAlignmentOf<word64>()));
229  ((word64 *)(void *)a)[0] = ((word64 *)(void *)b)[0] ^ ((word64 *)(void *)c)[0];
230  ((word64 *)(void *)a)[1] = ((word64 *)(void *)b)[1] ^ ((word64 *)(void *)c)[1];
231 }
232 
233 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
234 CRYPTOPP_ALIGN_DATA(16)
235 static const word64 s_clmulConstants64[] = {
236  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
237  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
238  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
239 
240 static const __m128i *s_clmulConstants = (const __m128i *)(const void *)s_clmulConstants64;
241 static const unsigned int s_clmulTableSizeInBlocks = 8;
242 
243 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2, const __m128i &r)
244 {
245  /*
246  The polynomial to be reduced is c0 * x^128 + c1 * x^64 + c2. c0t below refers to the most
247  significant half of c0 as a polynomial, which, due to GCM's bit reflection, are in the
248  rightmost bit positions, and the lowest byte addresses.
249 
250  c1 ^= c0t * 0xc200000000000000
251  c2t ^= c0t
252  t = shift (c1t ^ c0b) left 1 bit
253  c2 ^= t * 0xe100000000000000
254  c2t ^= c1b
255  shift c2 left 1 bit and xor in lowest bit of c1t
256  */
257 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301
258  c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
259 #else
260  c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
261 #endif
262  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
263  c0 = _mm_srli_si128(c0, 8);
264  c0 = _mm_xor_si128(c0, c1);
265  c0 = _mm_slli_epi64(c0, 1);
266  c0 = _mm_clmulepi64_si128(c0, r, 0);
267  c2 = _mm_xor_si128(c2, c0);
268  c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
269  c1 = _mm_unpacklo_epi64(c1, c2);
270  c1 = _mm_srli_epi64(c1, 63);
271  c2 = _mm_slli_epi64(c2, 1);
272  return _mm_xor_si128(c2, c1);
273 }
274 
275 inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r)
276 {
277  const __m128i c0 = _mm_clmulepi64_si128(x,h,0);
278  const __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
279  const __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
280 
281  return CLMUL_Reduce(c0, c1, c2, r);
282 }
283 #endif
284 
285 #if CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
286 
287 CRYPTOPP_ALIGN_DATA(16)
288 static const word64 s_clmulConstants64[] = {
289  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000), // Used for ARM and x86; polynomial coefficients
290  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607), // Unused for ARM; used for x86 _mm_shuffle_epi8
291  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f) // Unused for ARM; used for x86 _mm_shuffle_epi8
292 };
293 
294 static const uint64x2_t *s_clmulConstants = (const uint64x2_t *)s_clmulConstants64;
295 static const unsigned int s_clmulTableSizeInBlocks = 8;
296 
297 inline uint64x2_t PMULL_Reduce(uint64x2_t c0, uint64x2_t c1, uint64x2_t c2, const uint64x2_t &r)
298 {
299  // See comments fo CLMUL_Reduce
300  c1 = veorq_u64(c1, VEXT_U8<8>(vdupq_n_u64(0), c0));
301  c1 = veorq_u64(c1, PMULL_01(c0, r));
302  c0 = VEXT_U8<8>(c0, vdupq_n_u64(0));
303  c0 = vshlq_n_u64(veorq_u64(c0, c1), 1);
304  c0 = PMULL_00(c0, r);
305  c2 = veorq_u64(c2, c0);
306  c2 = veorq_u64(c2, VEXT_U8<8>(c1, vdupq_n_u64(0)));
307  c1 = vshrq_n_u64(vcombine_u64(vget_low_u64(c1), vget_low_u64(c2)), 63);
308  c2 = vshlq_n_u64(c2, 1);
309 
310  return veorq_u64(c2, c1);
311 }
312 
313 inline uint64x2_t PMULL_GF_Mul(const uint64x2_t &x, const uint64x2_t &h, const uint64x2_t &r)
314 {
315  const uint64x2_t c0 = PMULL_00(x, h);
316  const uint64x2_t c1 = veorq_u64(PMULL_10(x, h), PMULL_01(x, h));
317  const uint64x2_t c2 = PMULL_11(x, h);
318 
319  return PMULL_Reduce(c0, c1, c2, r);
320 }
321 #endif
322 
323 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs &params)
324 {
325  BlockCipher &blockCipher = AccessBlockCipher();
326  blockCipher.SetKey(userKey, keylength, params);
327 
328  if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
329  throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
330 
331  int tableSize, i, j, k;
332 
333 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
334  if (HasCLMUL())
335  {
336  // Avoid "parameter not used" error and suppress Coverity finding
337  (void)params.GetIntValue(Name::TableSize(), tableSize);
338  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
339  }
340  else
341 #elif CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
342  if (HasPMULL())
343  {
344  // Avoid "parameter not used" error and suppress Coverity finding
345  (void)params.GetIntValue(Name::TableSize(), tableSize);
346  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
347  }
348  else
349 #endif
350  {
351  if (params.GetIntValue(Name::TableSize(), tableSize))
352  tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
353  else
354  tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
355 
356 #if defined(_MSC_VER) && (_MSC_VER < 1400)
357  // VC 2003 workaround: compiler generates bad code for 64K tables
358  tableSize = 2*1024;
359 #endif
360  }
361 
362  m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
363  byte *table = MulTable();
364  byte *hashKey = HashKey();
365  memset(hashKey, 0, REQUIRED_BLOCKSIZE);
366  blockCipher.ProcessBlock(hashKey);
367 
368 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
369  if (HasCLMUL())
370  {
371  const __m128i r = s_clmulConstants[0];
372  __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)(void *)hashKey), s_clmulConstants[1]);
373  __m128i h = h0;
374 
375  for (i=0; i<tableSize; i+=32)
376  {
377  __m128i h1 = CLMUL_GF_Mul(h, h0, r);
378  _mm_storel_epi64((__m128i *)(void *)(table+i), h);
379  _mm_storeu_si128((__m128i *)(void *)(table+i+16), h1);
380  _mm_storeu_si128((__m128i *)(void *)(table+i+8), h);
381  _mm_storel_epi64((__m128i *)(void *)(table+i+8), h1);
382  h = CLMUL_GF_Mul(h1, h0, r);
383  }
384 
385  return;
386  }
387 #elif CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
388  if (HasPMULL())
389  {
390  const uint64x2_t r = s_clmulConstants[0];
391  const uint64x2_t t = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(hashKey)));
392  const uint64x2_t h0 = vextq_u64(t, t, 1);
393 
394  uint64x2_t h = h0;
395  for (i=0; i<tableSize-32; i+=32)
396  {
397  const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
398  vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
399  vst1q_u64((uint64_t *)(table+i+16), h1);
400  vst1q_u64((uint64_t *)(table+i+8), h);
401  vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
402  h = PMULL_GF_Mul(h1, h0, r);
403  }
404 
405  const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
406  vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
407  vst1q_u64((uint64_t *)(table+i+16), h1);
408  vst1q_u64((uint64_t *)(table+i+8), h);
409  vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
410 
411  return;
412  }
413 #endif
414 
415  word64 V0, V1;
416  typedef BlockGetAndPut<word64, BigEndian> Block;
417  Block::Get(hashKey)(V0)(V1);
418 
419  if (tableSize == 64*1024)
420  {
421  for (i=0; i<128; i++)
422  {
423  k = i%8;
424  Block::Put(NULLPTR, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
425 
426  int x = (int)V1 & 1;
427  V1 = (V1>>1) | (V0<<63);
428  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
429  }
430 
431  for (i=0; i<16; i++)
432  {
433  memset(table+i*256*16, 0, 16);
434 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
435  if (HasSSE2())
436  for (j=2; j<=0x80; j*=2)
437  for (k=1; k<j; k++)
438  SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
439  else
440 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
441  if (HasNEON())
442  for (j=2; j<=0x80; j*=2)
443  for (k=1; k<j; k++)
444  NEON_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
445  else
446 #endif
447  for (j=2; j<=0x80; j*=2)
448  for (k=1; k<j; k++)
449  Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
450  }
451  }
452  else
453  {
454  if (!s_reductionTableInitialized)
455  {
456  s_reductionTable[0] = 0;
457  word16 x = 0x01c2;
458  s_reductionTable[1] = ByteReverse(x);
459  for (unsigned int ii=2; ii<=0x80; ii*=2)
460  {
461  x <<= 1;
462  s_reductionTable[ii] = ByteReverse(x);
463  for (unsigned int jj=1; jj<ii; jj++)
464  s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
465  }
466  s_reductionTableInitialized = true;
467  }
468 
469  for (i=0; i<128-24; i++)
470  {
471  k = i%32;
472  if (k < 4)
473  Block::Put(NULLPTR, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
474  else if (k < 8)
475  Block::Put(NULLPTR, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
476 
477  int x = (int)V1 & 1;
478  V1 = (V1>>1) | (V0<<63);
479  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
480  }
481 
482  for (i=0; i<4; i++)
483  {
484  memset(table+i*256, 0, 16);
485  memset(table+1024+i*256, 0, 16);
486 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
487  if (HasSSE2())
488  for (j=2; j<=8; j*=2)
489  for (k=1; k<j; k++)
490  {
491  SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
492  SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
493  }
494  else
495 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
496  if (HasNEON())
497  for (j=2; j<=8; j*=2)
498  for (k=1; k<j; k++)
499  {
500  NEON_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
501  NEON_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
502  }
503  else
504 #endif
505  for (j=2; j<=8; j*=2)
506  for (k=1; k<j; k++)
507  {
508  Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
509  Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
510  }
511  }
512  }
513 }
514 
515 inline void GCM_Base::ReverseHashBufferIfNeeded()
516 {
517 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
518  if (HasCLMUL())
519  {
520  __m128i &x = *(__m128i *)(void *)HashBuffer();
521  x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
522  }
523 #elif CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
524  if (HasPMULL())
525  {
527  {
528  const uint8x16_t x = vrev64q_u8(vld1q_u8(HashBuffer()));
529  vst1q_u8(HashBuffer(), vextq_u8(x, x, 8));
530  }
531  }
532 #endif
533 }
534 
535 void GCM_Base::Resync(const byte *iv, size_t len)
536 {
537  BlockCipher &cipher = AccessBlockCipher();
538  byte *hashBuffer = HashBuffer();
539 
540  if (len == 12)
541  {
542  memcpy(hashBuffer, iv, len);
543  memset(hashBuffer+len, 0, 3);
544  hashBuffer[len+3] = 1;
545  }
546  else
547  {
548  size_t origLen = len;
549  memset(hashBuffer, 0, HASH_BLOCKSIZE);
550 
551  if (len >= HASH_BLOCKSIZE)
552  {
553  len = GCM_Base::AuthenticateBlocks(iv, len);
554  iv += (origLen - len);
555  }
556 
557  if (len > 0)
558  {
559  memcpy(m_buffer, iv, len);
560  memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
561  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
562  }
563 
564  PutBlock<word64, BigEndian, true>(NULLPTR, m_buffer)(0)(origLen*8);
565  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
566 
567  ReverseHashBufferIfNeeded();
568  }
569 
570  if (m_state >= State_IVSet)
571  m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
572  else
573  m_ctr.SetCipherWithIV(cipher, hashBuffer);
574 
575  m_ctr.Seek(HASH_BLOCKSIZE);
576 
577  memset(hashBuffer, 0, HASH_BLOCKSIZE);
578 }
579 
580 unsigned int GCM_Base::OptimalDataAlignment() const
581 {
582  return
583 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
584  HasSSE2() ? 16 :
585 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
586  HasNEON() ? 16 :
587 #endif
588  GetBlockCipher().OptimalDataAlignment();
589 }
590 
591 #if CRYPTOPP_MSC_VERSION
592 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
593 #endif
594 
595 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
596 
597 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
598 extern "C" {
599 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
600 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
601 }
602 #endif
603 
604 #ifndef CRYPTOPP_GENERATE_X64_MASM
605 
606 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
607 {
608 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
609  if (HasCLMUL())
610  {
611  const __m128i *table = (const __m128i *)(const void *)MulTable();
612  __m128i x = _mm_load_si128((__m128i *)(void *)HashBuffer());
613  const __m128i r = s_clmulConstants[0], mask1 = s_clmulConstants[1], mask2 = s_clmulConstants[2];
614 
615  while (len >= 16)
616  {
617  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
618  __m128i d1, d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-1)*16)), mask2);
619  __m128i c0 = _mm_setzero_si128();
620  __m128i c1 = _mm_setzero_si128();
621  __m128i c2 = _mm_setzero_si128();
622 
623  while (true)
624  {
625  __m128i h0 = _mm_load_si128(table+i);
626  __m128i h1 = _mm_load_si128(table+i+1);
627  __m128i h2 = _mm_xor_si128(h0, h1);
628 
629  if (++i == s)
630  {
631  d1 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), mask1);
632  d1 = _mm_xor_si128(d1, x);
633  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d1, h0, 0));
634  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d1, h1, 1));
635  d1 = _mm_xor_si128(d1, _mm_shuffle_epi32(d1, _MM_SHUFFLE(1, 0, 3, 2)));
636  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d1, h2, 0));
637  break;
638  }
639 
640  d1 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), mask2);
641  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
642  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d1, h1, 1));
643  d2 = _mm_xor_si128(d2, d1);
644  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h2, 1));
645 
646  if (++i == s)
647  {
648  d1 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), mask1);
649  d1 = _mm_xor_si128(d1, x);
650  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d1, h0, 0x10));
651  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d1, h1, 0x11));
652  d1 = _mm_xor_si128(d1, _mm_shuffle_epi32(d1, _MM_SHUFFLE(1, 0, 3, 2)));
653  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d1, h2, 0x10));
654  break;
655  }
656 
657  d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), mask1);
658  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d1, h0, 0x10));
659  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
660  d1 = _mm_xor_si128(d1, d2);
661  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d1, h2, 0x10));
662  }
663  data += s*16;
664  len -= s*16;
665 
666  c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
667  x = CLMUL_Reduce(c0, c1, c2, r);
668  }
669 
670  _mm_store_si128((__m128i *)(void *)HashBuffer(), x);
671  return len;
672  }
673 #elif CRYPTOPP_BOOL_ARM_PMULL_AVAILABLE
674  if (HasPMULL())
675  {
676  const uint64x2_t *table = (const uint64x2_t *)MulTable();
677  uint64x2_t x = vreinterpretq_u64_u8(vld1q_u8(HashBuffer()));
678  const uint64x2_t r = s_clmulConstants[0];
679 
680  while (len >= 16)
681  {
682  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
683  uint64x2_t d1, d2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(data+(s-1)*16)));
684  uint64x2_t c0 = vdupq_n_u64(0);
685  uint64x2_t c1 = vdupq_n_u64(0);
686  uint64x2_t c2 = vdupq_n_u64(0);
687 
688  while (true)
689  {
690  const uint64x2_t h0 = vld1q_u64((const uint64_t*)(table+i));
691  const uint64x2_t h1 = vld1q_u64((const uint64_t*)(table+i+1));
692  const uint64x2_t h2 = veorq_u64(h0, h1);
693 
694  if (++i == s)
695  {
696  const uint64x2_t t1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(data)));
697  d1 = veorq_u64(vextq_u64(t1, t1, 1), x);
698  c0 = veorq_u64(c0, PMULL_00(d1, h0));
699  c2 = veorq_u64(c2, PMULL_10(d1, h1));
700  d1 = veorq_u64(d1, (uint64x2_t)vcombine_u32(vget_high_u32(vreinterpretq_u32_u64(d1)),
701  vget_low_u32(vreinterpretq_u32_u64(d1))));
702  c1 = veorq_u64(c1, PMULL_00(d1, h2));
703 
704  break;
705  }
706 
707  d1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(data+(s-i)*16-8)));
708  c0 = veorq_u64(c0, PMULL_10(d2, h0));
709  c2 = veorq_u64(c2, PMULL_10(d1, h1));
710  d2 = veorq_u64(d2, d1);
711  c1 = veorq_u64(c1, PMULL_10(d2, h2));
712 
713  if (++i == s)
714  {
715  const uint64x2_t t2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(data)));
716  d1 = veorq_u64(vextq_u64(t2, t2, 1), x);
717  c0 = veorq_u64(c0, PMULL_01(d1, h0));
718  c2 = veorq_u64(c2, PMULL_11(d1, h1));
719  d1 = veorq_u64(d1, (uint64x2_t)vcombine_u32(vget_high_u32(vreinterpretq_u32_u64(d1)),
720  vget_low_u32(vreinterpretq_u32_u64(d1))));
721  c1 = veorq_u64(c1, PMULL_01(d1, h2));
722 
723  break;
724  }
725 
726  const uint64x2_t t3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(data+(s-i)*16-8)));
727  d2 = vextq_u64(t3, t3, 1);
728  c0 = veorq_u64(c0, PMULL_01(d1, h0));
729  c2 = veorq_u64(c2, PMULL_01(d2, h1));
730  d1 = veorq_u64(d1, d2);
731  c1 = veorq_u64(c1, PMULL_01(d1, h2));
732  }
733  data += s*16;
734  len -= s*16;
735 
736  c1 = veorq_u64(veorq_u64(c1, c0), c2);
737  x = PMULL_Reduce(c0, c1, c2, r);
738  }
739 
740  vst1q_u64((uint64_t *)HashBuffer(), x);
741  return len;
742 }
743 #endif
744 
746  word64 *hashBuffer = (word64 *)(void *)HashBuffer();
747  CRYPTOPP_ASSERT(IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
748 
749  switch (2*(m_buffer.size()>=64*1024)
750 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
751  + HasSSE2()
752 //#elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
753 // + HasNEON()
754 #endif
755  )
756  {
757  case 0: // non-SSE2 and 2K tables
758  {
759  byte *table = MulTable();
760  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
761 
762  do
763  {
764  word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
765  Block::Get(data)(y0)(y1);
766  x0 ^= y0;
767  x1 ^= y1;
768 
769  data += HASH_BLOCKSIZE;
770  len -= HASH_BLOCKSIZE;
771 
772  #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(table+(a*1024)+(b*256)+c+d*8)
773 
774  #ifdef IS_LITTLE_ENDIAN
775  #if CRYPTOPP_BOOL_SLOW_WORD64
776  word32 z0 = (word32)x0;
777  word32 z1 = (word32)(x0>>32);
778  word32 z2 = (word32)x1;
779  word32 z3 = (word32)(x1>>32);
780  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
781  #else
782  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
783  #endif
784  #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
785  #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
786  #else
787  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
788  #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
789  #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
790  #endif
791 
792  #define GF_MUL_32BY128(op, a, b, c) \
793  a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0); \
794  a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1); \
795  b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0); \
796  b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1); \
797  c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0); \
798  c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1); \
799  d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0); \
800  d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1); \
801 
802  GF_MUL_32BY128(=, 0, 0, 0)
803  GF_MUL_32BY128(^=, 0, 1, 1)
804  GF_MUL_32BY128(^=, 1, 0, 2)
805  GF_MUL_32BY128(^=, 1, 1, 3)
806 
807  word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
808  GF_SHIFT_8(d)
809  c0 ^= d0; c1 ^= d1;
810  r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
811  GF_SHIFT_8(c)
812  b0 ^= c0; b1 ^= c1;
813  r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
814  GF_SHIFT_8(b)
815  a0 ^= b0; a1 ^= b1;
816  a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
817  x0 = a0; x1 = a1;
818  }
819  while (len >= HASH_BLOCKSIZE);
820 
821  hashBuffer[0] = x0; hashBuffer[1] = x1;
822  return len;
823  }
824 
825  case 2: // non-SSE2 and 64K tables
826  {
827  byte *table = MulTable();
828  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
829 
830  do
831  {
832  word64 y0, y1, a0, a1;
833  Block::Get(data)(y0)(y1);
834  x0 ^= y0;
835  x1 ^= y1;
836 
837  data += HASH_BLOCKSIZE;
838  len -= HASH_BLOCKSIZE;
839 
840  #undef READ_TABLE_WORD64_COMMON
841  #undef READ_TABLE_WORD64
842 
843  #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(table+(a)*256*16+(c)+(d)*8)
844 
845  #ifdef IS_LITTLE_ENDIAN
846  #if CRYPTOPP_BOOL_SLOW_WORD64
847  word32 z0 = (word32)x0;
848  word32 z1 = (word32)(x0>>32);
849  word32 z2 = (word32)x1;
850  word32 z3 = (word32)(x1>>32);
851  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
852  #else
853  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
854  #endif
855  #else
856  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
857  #endif
858 
859  #define GF_MUL_8BY128(op, b, c, d) \
860  a0 op READ_TABLE_WORD64(b, c, d, 0);\
861  a1 op READ_TABLE_WORD64(b, c, d, 1);\
862 
863  GF_MUL_8BY128(=, 0, 0, 0)
864  GF_MUL_8BY128(^=, 0, 0, 1)
865  GF_MUL_8BY128(^=, 0, 0, 2)
866  GF_MUL_8BY128(^=, 0, 0, 3)
867  GF_MUL_8BY128(^=, 0, 1, 0)
868  GF_MUL_8BY128(^=, 0, 1, 1)
869  GF_MUL_8BY128(^=, 0, 1, 2)
870  GF_MUL_8BY128(^=, 0, 1, 3)
871  GF_MUL_8BY128(^=, 1, 2, 0)
872  GF_MUL_8BY128(^=, 1, 2, 1)
873  GF_MUL_8BY128(^=, 1, 2, 2)
874  GF_MUL_8BY128(^=, 1, 2, 3)
875  GF_MUL_8BY128(^=, 1, 3, 0)
876  GF_MUL_8BY128(^=, 1, 3, 1)
877  GF_MUL_8BY128(^=, 1, 3, 2)
878  GF_MUL_8BY128(^=, 1, 3, 3)
879 
880  x0 = a0; x1 = a1;
881  }
882  while (len >= HASH_BLOCKSIZE);
883 
884  hashBuffer[0] = x0; hashBuffer[1] = x1;
885  return len;
886  }
887 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
888 
889 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
890  case 1: // SSE2 and 2K tables
891  GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
892  return len % 16;
893  case 3: // SSE2 and 64K tables
894  GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
895  return len % 16;
896 #endif
897 
898 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
899  case 1: // SSE2 and 2K tables
900  {
901  #ifdef __GNUC__
902  __asm__ __volatile__
903  (
904  INTEL_NOPREFIX
905  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
906  ALIGN 8
907  GCM_AuthenticateBlocks_2K PROC FRAME
908  rex_push_reg rsi
909  push_reg rdi
910  push_reg rbx
911  .endprolog
912  mov rsi, r8
913  mov r11, r9
914  #else
915  AS2( mov WORD_REG(cx), data )
916  AS2( mov WORD_REG(dx), len )
917  AS2( mov WORD_REG(si), hashBuffer )
918  AS2( shr WORD_REG(dx), 4 )
919  #endif
920 
921  #if CRYPTOPP_BOOL_X32
922  AS1(push rbx)
923  AS1(push rbp)
924  #else
925  AS_PUSH_IF86( bx)
926  AS_PUSH_IF86( bp)
927  #endif
928 
929  #ifdef __GNUC__
930  AS2( mov AS_REG_7, WORD_REG(di))
931  #elif CRYPTOPP_BOOL_X86
932  AS2( lea AS_REG_7, s_reductionTable)
933  #endif
934 
935  AS2( movdqa xmm0, [WORD_REG(si)] )
936 
937  #define MUL_TABLE_0 WORD_REG(si) + 32
938  #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
939  #define RED_TABLE AS_REG_7
940 
941  ASL(0)
942  AS2( movdqu xmm4, [WORD_REG(cx)] )
943  AS2( pxor xmm0, xmm4 )
944 
945  AS2( movd ebx, xmm0 )
946  AS2( mov eax, AS_HEX(f0f0f0f0) )
947  AS2( and eax, ebx )
948  AS2( shl ebx, 4 )
949  AS2( and ebx, AS_HEX(f0f0f0f0) )
950  AS2( movzx edi, ah )
951  AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
952  AS2( movzx edi, al )
953  AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
954  AS2( shr eax, 16 )
955  AS2( movzx edi, ah )
956  AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
957  AS2( movzx edi, al )
958  AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
959 
960  #define SSE2_MUL_32BITS(i) \
961  AS2( psrldq xmm0, 4 )\
962  AS2( movd eax, xmm0 )\
963  AS2( and eax, AS_HEX(f0f0f0f0) )\
964  AS2( movzx edi, bh )\
965  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
966  AS2( movzx edi, bl )\
967  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
968  AS2( shr ebx, 16 )\
969  AS2( movzx edi, bh )\
970  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
971  AS2( movzx edi, bl )\
972  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
973  AS2( movd ebx, xmm0 )\
974  AS2( shl ebx, 4 )\
975  AS2( and ebx, AS_HEX(f0f0f0f0) )\
976  AS2( movzx edi, ah )\
977  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
978  AS2( movzx edi, al )\
979  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
980  AS2( shr eax, 16 )\
981  AS2( movzx edi, ah )\
982  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
983  AS2( movzx edi, al )\
984  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
985 
986  SSE2_MUL_32BITS(1)
987  SSE2_MUL_32BITS(2)
988  SSE2_MUL_32BITS(3)
989 
990  AS2( movzx edi, bh )
991  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
992  AS2( movzx edi, bl )
993  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
994  AS2( shr ebx, 16 )
995  AS2( movzx edi, bh )
996  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
997  AS2( movzx edi, bl )
998  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
999 
1000  AS2( movdqa xmm0, xmm3 )
1001  AS2( pslldq xmm3, 1 )
1002  AS2( pxor xmm2, xmm3 )
1003  AS2( movdqa xmm1, xmm2 )
1004  AS2( pslldq xmm2, 1 )
1005  AS2( pxor xmm5, xmm2 )
1006 
1007  AS2( psrldq xmm0, 15 )
1008 #if USE_MOVD_REG32
1009  AS2( movd edi, xmm0 )
1010 #elif USE_MOV_REG32_OR_REG64
1011  AS2( mov WORD_REG(di), xmm0 )
1012 #else // GNU Assembler
1013  AS2( movd WORD_REG(di), xmm0 )
1014 #endif
1015  AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
1016  AS2( shl eax, 8 )
1017 
1018  AS2( movdqa xmm0, xmm5 )
1019  AS2( pslldq xmm5, 1 )
1020  AS2( pxor xmm4, xmm5 )
1021 
1022  AS2( psrldq xmm1, 15 )
1023 #if USE_MOVD_REG32
1024  AS2( movd edi, xmm1 )
1025 #elif USE_MOV_REG32_OR_REG64
1026  AS2( mov WORD_REG(di), xmm1 )
1027 #else
1028  AS2( movd WORD_REG(di), xmm1 )
1029 #endif
1030  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
1031  AS2( shl eax, 8 )
1032 
1033  AS2( psrldq xmm0, 15 )
1034 #if USE_MOVD_REG32
1035  AS2( movd edi, xmm0 )
1036 #elif USE_MOV_REG32_OR_REG64
1037  AS2( mov WORD_REG(di), xmm0 )
1038 #else
1039  AS2( movd WORD_REG(di), xmm0 )
1040 #endif
1041  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
1042 
1043  AS2( movd xmm0, eax )
1044  AS2( pxor xmm0, xmm4 )
1045 
1046  AS2( add WORD_REG(cx), 16 )
1047  AS2( sub WORD_REG(dx), 1 )
1048  ATT_NOPREFIX
1049  ASJ( jnz, 0, b )
1050  INTEL_NOPREFIX
1051  AS2( movdqa [WORD_REG(si)], xmm0 )
1052 
1053  #if CRYPTOPP_BOOL_X32
1054  AS1(pop rbp)
1055  AS1(pop rbx)
1056  #else
1057  AS_POP_IF86( bp)
1058  AS_POP_IF86( bx)
1059  #endif
1060 
1061  #ifdef __GNUC__
1062  ATT_PREFIX
1063  :
1064  : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
1065  : "memory", "cc", "%eax"
1066  #if CRYPTOPP_BOOL_X64
1067  , "%ebx", "%r11"
1068  #endif
1069  );
1070  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
1071  pop rbx
1072  pop rdi
1073  pop rsi
1074  ret
1075  GCM_AuthenticateBlocks_2K ENDP
1076  #endif
1077 
1078  return len%16;
1079  }
1080  case 3: // SSE2 and 64K tables
1081  {
1082  #ifdef __GNUC__
1083  __asm__ __volatile__
1084  (
1085  INTEL_NOPREFIX
1086  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
1087  ALIGN 8
1088  GCM_AuthenticateBlocks_64K PROC FRAME
1089  rex_push_reg rsi
1090  push_reg rdi
1091  .endprolog
1092  mov rsi, r8
1093  #else
1094  AS2( mov WORD_REG(cx), data )
1095  AS2( mov WORD_REG(dx), len )
1096  AS2( mov WORD_REG(si), hashBuffer )
1097  AS2( shr WORD_REG(dx), 4 )
1098  #endif
1099 
1100  AS2( movdqa xmm0, [WORD_REG(si)] )
1101 
1102  #undef MUL_TABLE
1103  #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
1104 
1105  ASL(1)
1106  AS2( movdqu xmm1, [WORD_REG(cx)] )
1107  AS2( pxor xmm1, xmm0 )
1108  AS2( pxor xmm0, xmm0 )
1109 
1110  #undef SSE2_MUL_32BITS
1111  #define SSE2_MUL_32BITS(i) \
1112  AS2( movd eax, xmm1 )\
1113  AS2( psrldq xmm1, 4 )\
1114  AS2( movzx edi, al )\
1115  AS2( add WORD_REG(di), WORD_REG(di) )\
1116  AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
1117  AS2( movzx edi, ah )\
1118  AS2( add WORD_REG(di), WORD_REG(di) )\
1119  AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
1120  AS2( shr eax, 16 )\
1121  AS2( movzx edi, al )\
1122  AS2( add WORD_REG(di), WORD_REG(di) )\
1123  AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
1124  AS2( movzx edi, ah )\
1125  AS2( add WORD_REG(di), WORD_REG(di) )\
1126  AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
1127 
1128  SSE2_MUL_32BITS(0)
1129  SSE2_MUL_32BITS(1)
1130  SSE2_MUL_32BITS(2)
1131  SSE2_MUL_32BITS(3)
1132 
1133  AS2( add WORD_REG(cx), 16 )
1134  AS2( sub WORD_REG(dx), 1 )
1135  ATT_NOPREFIX
1136  ASJ( jnz, 1, b )
1137  INTEL_NOPREFIX
1138  AS2( movdqa [WORD_REG(si)], xmm0 )
1139 
1140  #ifdef __GNUC__
1141  ATT_PREFIX
1142  :
1143  : "c" (data), "d" (len/16), "S" (hashBuffer)
1144  : "memory", "cc", "%edi", "%eax"
1145  );
1146  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
1147  pop rdi
1148  pop rsi
1149  ret
1150  GCM_AuthenticateBlocks_64K ENDP
1151  #endif
1152 
1153  return len%16;
1154  }
1155 #endif
1156 #ifndef CRYPTOPP_GENERATE_X64_MASM
1157  }
1158 
1159  return len%16;
1160 }
1161 
1162 void GCM_Base::AuthenticateLastHeaderBlock()
1163 {
1164  if (m_bufferedDataLength > 0)
1165  {
1166  memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
1167  m_bufferedDataLength = 0;
1168  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1169  }
1170 }
1171 
1172 void GCM_Base::AuthenticateLastConfidentialBlock()
1173 {
1174  GCM_Base::AuthenticateLastHeaderBlock();
1175  PutBlock<word64, BigEndian, true>(NULLPTR, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
1176  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1177 }
1178 
1179 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
1180 {
1181  m_ctr.Seek(0);
1182  ReverseHashBufferIfNeeded();
1183  m_ctr.ProcessData(mac, HashBuffer(), macSize);
1184 }
1185 
1186 NAMESPACE_END
1187 
1188 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
1189 #endif
An invalid argument was detected.
Definition: cryptlib.h:194
virtual void SetKey(const byte *key, size_t length, const NameValuePairs &params=g_nullNameValuePairs)
Sets or reset the key of this object.
Definition: cryptlib.cpp:71
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: gcm.cpp:580
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
Definition: misc.h:1114
Library configuration file.
Access a block of memory.
Definition: misc.h:2345
virtual unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: cryptlib.cpp:201
byte order is little-endian
Definition: cryptlib.h:136
Polynomial with Coefficients in GF(2)
Definition: gf2n.h:21
Interface for one direction (encryption or decryption) of a block cipher.
Definition: cryptlib.h:1099
Use a table with 64K entries.
Definition: gcm.h:21
virtual unsigned int BlockSize() const =0
Provides the block size of the cipher.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1036
const char * TableSize()
int, in bytes
Definition: argnames.h:81
bool HasCLMUL()
Determines Carryless Multiply availability.
Definition: cpu.h:225
void ProcessBlock(const byte *inBlock, byte *outBlock) const
Encrypt or decrypt a block.
Definition: cryptlib.h:764
const T1 UnsignedMin(const T1 &a, const T2 &b)
Safe comparison of values that could be neagtive and incorrectly promoted.
Definition: misc.h:552
std::string AlgorithmName() const
Provides the name of this algorithm.
Definition: gcm.h:31
byte order is big-endian
Definition: cryptlib.h:138
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:60
const char * BlockSize()
int, in bytes
Definition: argnames.h:27
Functions for CPU features and intrinsics.
bool HasSSE2()
Determines SSE2 availability.
Definition: cpu.h:180
GCM block cipher mode of operation.
ByteOrder GetNativeByteOrder()
Returns NativeByteOrder as an enumerated ByteOrder value.
Definition: misc.h:1069
byte GetByte(size_t n) const
return the n-th byte
Definition: gf2n.cpp:83
Access a block of memory.
Definition: misc.h:2307
Crypto++ library namespace.
bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
Definition: cryptlib.h:379
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
Definition: misc.h:1766
Interface for retrieving values given their names.
Definition: cryptlib.h:285