Crypto++  5.6.3
Free C++ class library of cryptographic schemes
gcm.cpp
1 // gcm.cpp - written and placed in the public domain by Wei Dai
2 
3 // use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
4 
5 #include "pch.h"
6 #include "config.h"
7 
8 #if CRYPTOPP_MSC_VERSION
9 # pragma warning(disable: 4189)
10 #endif
11 
12 #ifndef CRYPTOPP_IMPORTS
13 #ifndef CRYPTOPP_GENERATE_X64_MASM
14 
15 // Clang 3.3 integrated assembler crash on Linux. MacPorts GCC compile error. SunCC crash under Sun Studio 12.5
16 #if (defined(CRYPTOPP_LLVM_CLANG_VERSION) && (CRYPTOPP_LLVM_CLANG_VERSION < 30400)) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER) || (__SUNPRO_CC == 0x5140)
17 # undef CRYPTOPP_X86_ASM_AVAILABLE
18 # undef CRYPTOPP_X32_ASM_AVAILABLE
19 # undef CRYPTOPP_X64_ASM_AVAILABLE
20 # undef CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
21 # undef CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE
22 # undef CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
23 # undef CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
24 # define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0
25 # define CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 0
26 # define CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 0
27 # define CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 0
28 #endif
29 
30 #include "gcm.h"
31 #include "cpu.h"
32 
33 NAMESPACE_BEGIN(CryptoPP)
34 
35 word16 GCM_Base::s_reductionTable[256];
36 volatile bool GCM_Base::s_reductionTableInitialized = false;
37 
38 void GCM_Base::GCTR::IncrementCounterBy256()
39 {
40  IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
41 }
42 
43 #if 0
44 // preserved for testing
45 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
46 {
47  word64 Z0=0, Z1=0, V0, V1;
48 
50  Block::Get(a)(V0)(V1);
51 
52  for (int i=0; i<16; i++)
53  {
54  for (int j=0x80; j!=0; j>>=1)
55  {
56  int x = b[i] & j;
57  Z0 ^= x ? V0 : 0;
58  Z1 ^= x ? V1 : 0;
59  x = (int)V1 & 1;
60  V1 = (V1>>1) | (V0<<63);
61  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
62  }
63  }
64  Block::Put(NULL, c)(Z0)(Z1);
65 }
66 
67 __m128i _mm_clmulepi64_si128(const __m128i &a, const __m128i &b, int i)
68 {
69  word64 A[1] = {ByteReverse(((word64*)&a)[i&1])};
70  word64 B[1] = {ByteReverse(((word64*)&b)[i>>4])};
71 
72  PolynomialMod2 pa((byte *)A, 8);
73  PolynomialMod2 pb((byte *)B, 8);
74  PolynomialMod2 c = pa*pb;
75 
76  __m128i output;
77  for (int i=0; i<16; i++)
78  ((byte *)&output)[i] = c.GetByte(i);
79  return output;
80 }
81 #endif
82 
83 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
84 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
85 {
86 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
87  assert(IsAlignedOn(a,GetAlignmentOf<__m128i>()));
88  assert(IsAlignedOn(b,GetAlignmentOf<__m128i>()));
89  assert(IsAlignedOn(c,GetAlignmentOf<__m128i>()));
90  *(__m128i *)(void *)a = _mm_xor_si128(*(__m128i *)(void *)b, *(__m128i *)(void *)c);
91 #else
92  asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
93 #endif
94 }
95 #endif
96 
97 inline static void Xor16(byte *a, const byte *b, const byte *c)
98 {
99  assert(IsAlignedOn(a,GetAlignmentOf<word64>()));
100  assert(IsAlignedOn(b,GetAlignmentOf<word64>()));
101  assert(IsAlignedOn(c,GetAlignmentOf<word64>()));
102  ((word64 *)(void *)a)[0] = ((word64 *)(void *)b)[0] ^ ((word64 *)(void *)c)[0];
103  ((word64 *)(void *)a)[1] = ((word64 *)(void *)b)[1] ^ ((word64 *)(void *)c)[1];
104 }
105 
106 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
107 CRYPTOPP_ALIGN_DATA(16)
108 static const word64 s_clmulConstants64[] = {
109  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
110  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
111  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
112 static const __m128i *s_clmulConstants = (const __m128i *)(const void *)s_clmulConstants64;
113 static const unsigned int s_clmulTableSizeInBlocks = 8;
114 
115 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2, const __m128i &r)
116 {
117  /*
118  The polynomial to be reduced is c0 * x^128 + c1 * x^64 + c2. c0t below refers to the most
119  significant half of c0 as a polynomial, which, due to GCM's bit reflection, are in the
120  rightmost bit positions, and the lowest byte addresses.
121 
122  c1 ^= c0t * 0xc200000000000000
123  c2t ^= c0t
124  t = shift (c1t ^ c0b) left 1 bit
125  c2 ^= t * 0xe100000000000000
126  c2t ^= c1b
127  shift c2 left 1 bit and xor in lowest bit of c1t
128  */
129 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301
130  c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
131 #else
132  c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
133 #endif
134  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
135  c0 = _mm_srli_si128(c0, 8);
136  c0 = _mm_xor_si128(c0, c1);
137  c0 = _mm_slli_epi64(c0, 1);
138  c0 = _mm_clmulepi64_si128(c0, r, 0);
139  c2 = _mm_xor_si128(c2, c0);
140  c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
141  c1 = _mm_unpacklo_epi64(c1, c2);
142  c1 = _mm_srli_epi64(c1, 63);
143  c2 = _mm_slli_epi64(c2, 1);
144  return _mm_xor_si128(c2, c1);
145 }
146 
147 inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r)
148 {
149  __m128i c0 = _mm_clmulepi64_si128(x,h,0);
150  __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
151  __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
152 
153  return CLMUL_Reduce(c0, c1, c2, r);
154 }
155 #endif
156 
157 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs &params)
158 {
159  BlockCipher &blockCipher = AccessBlockCipher();
160  blockCipher.SetKey(userKey, keylength, params);
161 
162  if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
163  throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
164 
165  int tableSize, i, j, k;
166 
167 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
168  if (HasCLMUL())
169  {
170  // Avoid "parameter not used" error and suppress Coverity finding
171  (void)params.GetIntValue(Name::TableSize(), tableSize);
172  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
173  }
174  else
175 #endif
176  {
177  if (params.GetIntValue(Name::TableSize(), tableSize))
178  tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
179  else
180  tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
181 
182 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
183  // VC 2003 workaround: compiler generates bad code for 64K tables
184  tableSize = 2*1024;
185 #endif
186  }
187 
188  m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
189  byte *table = MulTable();
190  byte *hashKey = HashKey();
191  memset(hashKey, 0, REQUIRED_BLOCKSIZE);
192  blockCipher.ProcessBlock(hashKey);
193 
194 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
195  if (HasCLMUL())
196  {
197  const __m128i r = s_clmulConstants[0];
198  __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)(void *)hashKey), s_clmulConstants[1]);
199  __m128i h = h0;
200 
201  for (i=0; i<tableSize; i+=32)
202  {
203  __m128i h1 = CLMUL_GF_Mul(h, h0, r);
204  _mm_storel_epi64((__m128i *)(void *)(table+i), h);
205  _mm_storeu_si128((__m128i *)(void *)(table+i+16), h1);
206  _mm_storeu_si128((__m128i *)(void *)(table+i+8), h);
207  _mm_storel_epi64((__m128i *)(void *)(table+i+8), h1);
208  h = CLMUL_GF_Mul(h1, h0, r);
209  }
210 
211  return;
212  }
213 #endif
214 
215  word64 V0, V1;
216  typedef BlockGetAndPut<word64, BigEndian> Block;
217  Block::Get(hashKey)(V0)(V1);
218 
219  if (tableSize == 64*1024)
220  {
221  for (i=0; i<128; i++)
222  {
223  k = i%8;
224  Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
225 
226  int x = (int)V1 & 1;
227  V1 = (V1>>1) | (V0<<63);
228  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
229  }
230 
231  for (i=0; i<16; i++)
232  {
233  memset(table+i*256*16, 0, 16);
234 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
235  if (HasSSE2())
236  for (j=2; j<=0x80; j*=2)
237  for (k=1; k<j; k++)
238  SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
239  else
240 #endif
241  for (j=2; j<=0x80; j*=2)
242  for (k=1; k<j; k++)
243  Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
244  }
245  }
246  else
247  {
248  if (!s_reductionTableInitialized)
249  {
250  s_reductionTable[0] = 0;
251  word16 x = 0x01c2;
252  s_reductionTable[1] = ByteReverse(x);
253  for (unsigned int ii=2; ii<=0x80; ii*=2)
254  {
255  x <<= 1;
256  s_reductionTable[ii] = ByteReverse(x);
257  for (unsigned int jj=1; jj<ii; jj++)
258  s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
259  }
260  s_reductionTableInitialized = true;
261  }
262 
263  for (i=0; i<128-24; i++)
264  {
265  k = i%32;
266  if (k < 4)
267  Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
268  else if (k < 8)
269  Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
270 
271  int x = (int)V1 & 1;
272  V1 = (V1>>1) | (V0<<63);
273  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
274  }
275 
276  for (i=0; i<4; i++)
277  {
278  memset(table+i*256, 0, 16);
279  memset(table+1024+i*256, 0, 16);
280 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
281  if (HasSSE2())
282  for (j=2; j<=8; j*=2)
283  for (k=1; k<j; k++)
284  {
285  SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
286  SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
287  }
288  else
289 #endif
290  for (j=2; j<=8; j*=2)
291  for (k=1; k<j; k++)
292  {
293  Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
294  Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
295  }
296  }
297  }
298 }
299 
300 inline void GCM_Base::ReverseHashBufferIfNeeded()
301 {
302 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
303  if (HasCLMUL())
304  {
305  __m128i &x = *(__m128i *)(void *)HashBuffer();
306  x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
307  }
308 #endif
309 }
310 
311 void GCM_Base::Resync(const byte *iv, size_t len)
312 {
313  BlockCipher &cipher = AccessBlockCipher();
314  byte *hashBuffer = HashBuffer();
315 
316  if (len == 12)
317  {
318  memcpy(hashBuffer, iv, len);
319  memset(hashBuffer+len, 0, 3);
320  hashBuffer[len+3] = 1;
321  }
322  else
323  {
324  size_t origLen = len;
325  memset(hashBuffer, 0, HASH_BLOCKSIZE);
326 
327  if (len >= HASH_BLOCKSIZE)
328  {
329  len = GCM_Base::AuthenticateBlocks(iv, len);
330  iv += (origLen - len);
331  }
332 
333  if (len > 0)
334  {
335  memcpy(m_buffer, iv, len);
336  memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
337  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
338  }
339 
340  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
341  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
342 
343  ReverseHashBufferIfNeeded();
344  }
345 
346  if (m_state >= State_IVSet)
347  m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
348  else
349  m_ctr.SetCipherWithIV(cipher, hashBuffer);
350 
351  m_ctr.Seek(HASH_BLOCKSIZE);
352 
353  memset(hashBuffer, 0, HASH_BLOCKSIZE);
354 }
355 
356 unsigned int GCM_Base::OptimalDataAlignment() const
357 {
358  return
359 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
360  HasSSE2() ? 16 :
361 #endif
362  GetBlockCipher().OptimalDataAlignment();
363 }
364 
365 #if CRYPTOPP_MSC_VERSION
366 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
367 #endif
368 
369 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
370 
371 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
372 extern "C" {
373 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
374 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
375 }
376 #endif
377 
378 #ifndef CRYPTOPP_GENERATE_X64_MASM
379 
380 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
381 {
382 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
383  if (HasCLMUL())
384  {
385  const __m128i *table = (const __m128i *)(const void *)MulTable();
386  __m128i x = _mm_load_si128((__m128i *)(void *)HashBuffer());
387  const __m128i r = s_clmulConstants[0], bswapMask = s_clmulConstants[1], bswapMask2 = s_clmulConstants[2];
388 
389  while (len >= 16)
390  {
391  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
392  __m128i d, d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-1)*16)), bswapMask2);;
393  __m128i c0 = _mm_setzero_si128();
394  __m128i c1 = _mm_setzero_si128();
395  __m128i c2 = _mm_setzero_si128();
396 
397  while (true)
398  {
399  __m128i h0 = _mm_load_si128(table+i);
400  __m128i h1 = _mm_load_si128(table+i+1);
401  __m128i h01 = _mm_xor_si128(h0, h1);
402 
403  if (++i == s)
404  {
405  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), bswapMask);
406  d = _mm_xor_si128(d, x);
407  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0));
408  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
409  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
410  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0));
411  break;
412  }
413 
414  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), bswapMask2);
415  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
416  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
417  d2 = _mm_xor_si128(d2, d);
418  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h01, 1));
419 
420  if (++i == s)
421  {
422  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), bswapMask);
423  d = _mm_xor_si128(d, x);
424  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
425  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 0x11));
426  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
427  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
428  break;
429  }
430 
431  d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), bswapMask);
432  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
433  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
434  d = _mm_xor_si128(d, d2);
435  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
436  }
437  data += s*16;
438  len -= s*16;
439 
440  c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
441  x = CLMUL_Reduce(c0, c1, c2, r);
442  }
443 
444  _mm_store_si128((__m128i *)(void *)HashBuffer(), x);
445  return len;
446  }
447 #endif
448 
450  word64 *hashBuffer = (word64 *)(void *)HashBuffer();
451  assert(IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
452 
453  switch (2*(m_buffer.size()>=64*1024)
454 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
455  + HasSSE2()
456 #endif
457  )
458  {
459  case 0: // non-SSE2 and 2K tables
460  {
461  byte *table = MulTable();
462  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
463 
464  do
465  {
466  word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
467  Block::Get(data)(y0)(y1);
468  x0 ^= y0;
469  x1 ^= y1;
470 
471  data += HASH_BLOCKSIZE;
472  len -= HASH_BLOCKSIZE;
473 
474  #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(table+(a*1024)+(b*256)+c+d*8)
475 
476  #ifdef IS_LITTLE_ENDIAN
477  #if CRYPTOPP_BOOL_SLOW_WORD64
478  word32 z0 = (word32)x0;
479  word32 z1 = (word32)(x0>>32);
480  word32 z2 = (word32)x1;
481  word32 z3 = (word32)(x1>>32);
482  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
483  #else
484  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
485  #endif
486  #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
487  #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
488  #else
489  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
490  #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
491  #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
492  #endif
493 
494  #define GF_MUL_32BY128(op, a, b, c) \
495  a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
496  a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
497  b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
498  b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
499  c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
500  c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
501  d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
502  d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
503 
504  GF_MUL_32BY128(=, 0, 0, 0)
505  GF_MUL_32BY128(^=, 0, 1, 1)
506  GF_MUL_32BY128(^=, 1, 0, 2)
507  GF_MUL_32BY128(^=, 1, 1, 3)
508 
509  word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
510  GF_SHIFT_8(d)
511  c0 ^= d0; c1 ^= d1;
512  r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
513  GF_SHIFT_8(c)
514  b0 ^= c0; b1 ^= c1;
515  r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
516  GF_SHIFT_8(b)
517  a0 ^= b0; a1 ^= b1;
519  x0 = a0; x1 = a1;
520  }
521  while (len >= HASH_BLOCKSIZE);
522 
523  hashBuffer[0] = x0; hashBuffer[1] = x1;
524  return len;
525  }
526 
527  case 2: // non-SSE2 and 64K tables
528  {
529  byte *table = MulTable();
530  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
531 
532  do
533  {
534  word64 y0, y1, a0, a1;
535  Block::Get(data)(y0)(y1);
536  x0 ^= y0;
537  x1 ^= y1;
538 
539  data += HASH_BLOCKSIZE;
540  len -= HASH_BLOCKSIZE;
541 
542  #undef READ_TABLE_WORD64_COMMON
543  #undef READ_TABLE_WORD64
544 
545  #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(table+(a)*256*16+(c)+(d)*8)
546 
547  #ifdef IS_LITTLE_ENDIAN
548  #if CRYPTOPP_BOOL_SLOW_WORD64
549  word32 z0 = (word32)x0;
550  word32 z1 = (word32)(x0>>32);
551  word32 z2 = (word32)x1;
552  word32 z3 = (word32)(x1>>32);
553  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
554  #else
555  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
556  #endif
557  #else
558  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
559  #endif
560 
561  #define GF_MUL_8BY128(op, b, c, d) \
562  a0 op READ_TABLE_WORD64(b, c, d, 0);\
563  a1 op READ_TABLE_WORD64(b, c, d, 1);\
564 
565  GF_MUL_8BY128(=, 0, 0, 0)
566  GF_MUL_8BY128(^=, 0, 0, 1)
567  GF_MUL_8BY128(^=, 0, 0, 2)
568  GF_MUL_8BY128(^=, 0, 0, 3)
569  GF_MUL_8BY128(^=, 0, 1, 0)
570  GF_MUL_8BY128(^=, 0, 1, 1)
571  GF_MUL_8BY128(^=, 0, 1, 2)
572  GF_MUL_8BY128(^=, 0, 1, 3)
573  GF_MUL_8BY128(^=, 1, 2, 0)
574  GF_MUL_8BY128(^=, 1, 2, 1)
575  GF_MUL_8BY128(^=, 1, 2, 2)
576  GF_MUL_8BY128(^=, 1, 2, 3)
577  GF_MUL_8BY128(^=, 1, 3, 0)
578  GF_MUL_8BY128(^=, 1, 3, 1)
579  GF_MUL_8BY128(^=, 1, 3, 2)
580  GF_MUL_8BY128(^=, 1, 3, 3)
581 
582  x0 = a0; x1 = a1;
583  }
584  while (len >= HASH_BLOCKSIZE);
585 
586  hashBuffer[0] = x0; hashBuffer[1] = x1;
587  return len;
588  }
589 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
590 
591 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
592  case 1: // SSE2 and 2K tables
593  GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
594  return len % 16;
595  case 3: // SSE2 and 64K tables
596  GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
597  return len % 16;
598 #endif
599 
600 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
601  case 1: // SSE2 and 2K tables
602  {
603  #ifdef __GNUC__
604  __asm__ __volatile__
605  (
606  INTEL_NOPREFIX
607  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
608  ALIGN 8
609  GCM_AuthenticateBlocks_2K PROC FRAME
610  rex_push_reg rsi
611  push_reg rdi
612  push_reg rbx
613  .endprolog
614  mov rsi, r8
615  mov r11, r9
616  #else
617  AS2( mov WORD_REG(cx), data )
618  AS2( mov WORD_REG(dx), len )
619  AS2( mov WORD_REG(si), hashBuffer )
620  AS2( shr WORD_REG(dx), 4 )
621  #endif
622 
623  #if CRYPTOPP_BOOL_X32
624  AS1(push rbx)
625  AS1(push rbp)
626  #else
627  AS_PUSH_IF86( bx)
628  AS_PUSH_IF86( bp)
629  #endif
630 
631  #ifdef __GNUC__
632  AS2( mov AS_REG_7, WORD_REG(di))
633  #elif CRYPTOPP_BOOL_X86
634  AS2( lea AS_REG_7, s_reductionTable)
635  #endif
636 
637  AS2( movdqa xmm0, [WORD_REG(si)] )
638 
639  #define MUL_TABLE_0 WORD_REG(si) + 32
640  #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
641  #define RED_TABLE AS_REG_7
642 
643  ASL(0)
644  AS2( movdqu xmm4, [WORD_REG(cx)] )
645  AS2( pxor xmm0, xmm4 )
646 
647  AS2( movd ebx, xmm0 )
648  AS2( mov eax, AS_HEX(f0f0f0f0) )
649  AS2( and eax, ebx )
650  AS2( shl ebx, 4 )
651  AS2( and ebx, AS_HEX(f0f0f0f0) )
652  AS2( movzx edi, ah )
653  AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
654  AS2( movzx edi, al )
655  AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
656  AS2( shr eax, 16 )
657  AS2( movzx edi, ah )
658  AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
659  AS2( movzx edi, al )
660  AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
661 
662  #define SSE2_MUL_32BITS(i) \
663  AS2( psrldq xmm0, 4 )\
664  AS2( movd eax, xmm0 )\
665  AS2( and eax, AS_HEX(f0f0f0f0) )\
666  AS2( movzx edi, bh )\
667  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
668  AS2( movzx edi, bl )\
669  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
670  AS2( shr ebx, 16 )\
671  AS2( movzx edi, bh )\
672  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
673  AS2( movzx edi, bl )\
674  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
675  AS2( movd ebx, xmm0 )\
676  AS2( shl ebx, 4 )\
677  AS2( and ebx, AS_HEX(f0f0f0f0) )\
678  AS2( movzx edi, ah )\
679  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
680  AS2( movzx edi, al )\
681  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
682  AS2( shr eax, 16 )\
683  AS2( movzx edi, ah )\
684  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
685  AS2( movzx edi, al )\
686  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
687 
688  SSE2_MUL_32BITS(1)
689  SSE2_MUL_32BITS(2)
690  SSE2_MUL_32BITS(3)
691 
692  AS2( movzx edi, bh )
693  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
694  AS2( movzx edi, bl )
695  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
696  AS2( shr ebx, 16 )
697  AS2( movzx edi, bh )
698  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
699  AS2( movzx edi, bl )
700  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
701 
702  AS2( movdqa xmm0, xmm3 )
703  AS2( pslldq xmm3, 1 )
704  AS2( pxor xmm2, xmm3 )
705  AS2( movdqa xmm1, xmm2 )
706  AS2( pslldq xmm2, 1 )
707  AS2( pxor xmm5, xmm2 )
708 
709  AS2( psrldq xmm0, 15 )
710 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
711  AS2( movd edi, xmm0 )
712 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
713  AS2( mov WORD_REG(di), xmm0 )
714 #else // GNU Assembler
715  AS2( movd WORD_REG(di), xmm0 )
716 #endif
717  AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
718  AS2( shl eax, 8 )
719 
720  AS2( movdqa xmm0, xmm5 )
721  AS2( pslldq xmm5, 1 )
722  AS2( pxor xmm4, xmm5 )
723 
724  AS2( psrldq xmm1, 15 )
725 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
726  AS2( movd edi, xmm1 )
727 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
728  AS2( mov WORD_REG(di), xmm1 )
729 #else
730  AS2( movd WORD_REG(di), xmm1 )
731 #endif
732  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
733  AS2( shl eax, 8 )
734 
735  AS2( psrldq xmm0, 15 )
736 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
737  AS2( movd edi, xmm0 )
738 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
739  AS2( mov WORD_REG(di), xmm0 )
740 #else
741  AS2( movd WORD_REG(di), xmm0 )
742 #endif
743  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
744 
745  AS2( movd xmm0, eax )
746  AS2( pxor xmm0, xmm4 )
747 
748  AS2( add WORD_REG(cx), 16 )
749  AS2( sub WORD_REG(dx), 1 )
750  ATT_NOPREFIX
751  ASJ( jnz, 0, b )
752  INTEL_NOPREFIX
753  AS2( movdqa [WORD_REG(si)], xmm0 )
754 
755  #if CRYPTOPP_BOOL_X32
756  AS1(pop rbp)
757  AS1(pop rbx)
758  #else
759  AS_POP_IF86( bp)
760  AS_POP_IF86( bx)
761  #endif
762 
763  #ifdef __GNUC__
764  ATT_PREFIX
765  :
766  : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
767  : "memory", "cc", "%eax"
768  #if CRYPTOPP_BOOL_X64
769  , "%ebx", "%r11"
770  #endif
771  );
772  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
773  pop rbx
774  pop rdi
775  pop rsi
776  ret
777  GCM_AuthenticateBlocks_2K ENDP
778  #endif
779 
780  return len%16;
781  }
782  case 3: // SSE2 and 64K tables
783  {
784  #ifdef __GNUC__
785  __asm__ __volatile__
786  (
787  INTEL_NOPREFIX
788  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
789  ALIGN 8
790  GCM_AuthenticateBlocks_64K PROC FRAME
791  rex_push_reg rsi
792  push_reg rdi
793  .endprolog
794  mov rsi, r8
795  #else
796  AS2( mov WORD_REG(cx), data )
797  AS2( mov WORD_REG(dx), len )
798  AS2( mov WORD_REG(si), hashBuffer )
799  AS2( shr WORD_REG(dx), 4 )
800  #endif
801 
802  AS2( movdqa xmm0, [WORD_REG(si)] )
803 
804  #undef MUL_TABLE
805  #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
806 
807  ASL(1)
808  AS2( movdqu xmm1, [WORD_REG(cx)] )
809  AS2( pxor xmm1, xmm0 )
810  AS2( pxor xmm0, xmm0 )
811 
812  #undef SSE2_MUL_32BITS
813  #define SSE2_MUL_32BITS(i) \
814  AS2( movd eax, xmm1 )\
815  AS2( psrldq xmm1, 4 )\
816  AS2( movzx edi, al )\
817  AS2( add WORD_REG(di), WORD_REG(di) )\
818  AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
819  AS2( movzx edi, ah )\
820  AS2( add WORD_REG(di), WORD_REG(di) )\
821  AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
822  AS2( shr eax, 16 )\
823  AS2( movzx edi, al )\
824  AS2( add WORD_REG(di), WORD_REG(di) )\
825  AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
826  AS2( movzx edi, ah )\
827  AS2( add WORD_REG(di), WORD_REG(di) )\
828  AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
829 
830  SSE2_MUL_32BITS(0)
831  SSE2_MUL_32BITS(1)
832  SSE2_MUL_32BITS(2)
833  SSE2_MUL_32BITS(3)
834 
835  AS2( add WORD_REG(cx), 16 )
836  AS2( sub WORD_REG(dx), 1 )
837  ATT_NOPREFIX
838  ASJ( jnz, 1, b )
839  INTEL_NOPREFIX
840  AS2( movdqa [WORD_REG(si)], xmm0 )
841 
842  #ifdef __GNUC__
843  ATT_PREFIX
844  :
845  : "c" (data), "d" (len/16), "S" (hashBuffer)
846  : "memory", "cc", "%edi", "%eax"
847  );
848  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
849  pop rdi
850  pop rsi
851  ret
852  GCM_AuthenticateBlocks_64K ENDP
853  #endif
854 
855  return len%16;
856  }
857 #endif
858 #ifndef CRYPTOPP_GENERATE_X64_MASM
859  }
860 
861  return len%16;
862 }
863 
864 void GCM_Base::AuthenticateLastHeaderBlock()
865 {
866  if (m_bufferedDataLength > 0)
867  {
868  memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
869  m_bufferedDataLength = 0;
870  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
871  }
872 }
873 
874 void GCM_Base::AuthenticateLastConfidentialBlock()
875 {
876  GCM_Base::AuthenticateLastHeaderBlock();
877  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
878  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
879 }
880 
881 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
882 {
883  m_ctr.Seek(0);
884  ReverseHashBufferIfNeeded();
885  m_ctr.ProcessData(mac, HashBuffer(), macSize);
886 }
887 
888 NAMESPACE_END
889 
890 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
891 #endif
GCM block cipher base implementation.
Definition: gcm.h:25
An invalid argument was detected.
Definition: cryptlib.h:182
virtual void SetKey(const byte *key, size_t length, const NameValuePairs &params=g_nullNameValuePairs)
Sets or reset the key of this object.
Definition: cryptlib.cpp:100
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
Definition: misc.h:994
virtual unsigned int BlockSize() const =0
Provides the block size of the cipher.
Library configuration file.
byte GetByte(size_t n) const
return the n-th byte
Definition: gf2n.cpp:77
Access a block of memory.
Definition: misc.h:2209
bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
Definition: cryptlib.h:371
byte order is little-endian
Definition: cryptlib.h:125
Polynomial with Coefficients in GF(2)
Definition: gf2n.h:18
Interface for one direction (encryption or decryption) of a block cipher.
Definition: cryptlib.h:1091
Use a table with 64K entries.
Definition: gcm.h:20
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:888
const char * TableSize()
int, in bytes
Definition: argnames.h:80
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: gcm.cpp:356
void ProcessBlock(const byte *inBlock, byte *outBlock) const
Encrypt or decrypt a block.
Definition: cryptlib.h:751
bool HasCLMUL()
Determines Carryless Multiply availability.
Definition: cpu.h:250
T ConditionalByteReverse(ByteOrder order, T value)
Reverses bytes in a value depending upon endianess.
Definition: misc.h:1812
const T1 UnsignedMin(const T1 &a, const T2 &b)
Safe comparison of values that could be neagtive and incorrectly promoted.
Definition: misc.h:486
const char * BlockSize()
int, in bytes
Definition: argnames.h:26
Functions for CPU features and intrinsics.
virtual unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: cryptlib.cpp:232
bool HasSSE2()
Determines SSE2 availability.
Definition: cpu.h:205
std::string AlgorithmName() const
Provides the name of this algorithm.
Definition: gcm.h:29
GCM block cipher mode of operation.
Access a block of memory.
Definition: misc.h:2171
Crypto++ library namespace.
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
Definition: misc.h:1668
Interface for retrieving values given their names.
Definition: cryptlib.h:277