Crypto++  7.0
Free C++ class library of cryptographic schemes
adv_simd.h
Go to the documentation of this file.
1 // adv_simd.h - written and placed in the public domain by Jeffrey Walton
2 
3 /// \file adv_simd.h
4 /// \brief Template for AdvancedProcessBlocks and SIMD processing
5 
6 // The SIMD based implementations for ciphers that use SSE, NEON and Power7
7 // have a commom pattern. Namely, they have a specialized implementation of
8 // AdvancedProcessBlocks which processes multiple block using hardware
9 // acceleration. After several implementations we noticed a lot of copy and
10 // paste occuring. adv_simd.h provides a template to avoid the copy and paste.
11 //
12 // There are 11 templates provided in this file. The number following the
13 // function name, 64 or 128, is the block size. The name following the block
14 // size is the arrangement and acceleration. For example 4x1_SSE means Intel
15 // SSE using two encrypt (or decrypt) functions: one that operates on 4 SIMD
16 // words, and one that operates on 1 SIMD words.
17 //
18 // The distinction between SIMD words versus cipher blocks is important
19 // because 64-bit ciphers use one SIMD word for two cipher blocks. For
20 // example, AdvancedProcessBlocks64_6x2_ALTIVEC operates on 6 and 2 SIMD
21 // words, which is 12 and 4 cipher blocks. The function will do the right
22 // thing even if there is only one 64-bit block to encrypt.
23 //
24 // * AdvancedProcessBlocks64_2x1_SSE
25 // * AdvancedProcessBlocks64_4x1_SSE
26 // * AdvancedProcessBlocks128_4x1_SSE
27 // * AdvancedProcessBlocks64_6x2_SSE
28 // * AdvancedProcessBlocks128_6x2_SSE
29 // * AdvancedProcessBlocks64_6x2_NEON
30 // * AdvancedProcessBlocks128_4x1_NEON
31 // * AdvancedProcessBlocks128_6x2_NEON
32 // * AdvancedProcessBlocks64_6x2_ALTIVEC
33 // * AdvancedProcessBlocks128_4x1_ALTIVEC
34 // * AdvancedProcessBlocks128_6x1_ALTIVEC
35 //
36 // If an arrangement ends in 2, like 6x2, then the template will handle the
37 // single block case by padding with 0's and using the two SIMD word
38 // function. This happens at most one time when processing multiple blocks.
39 // The extra processing of a zero block is trivial and worth the tradeoff.
40 //
41 // The MAYBE_CONST macro present on x86 is a SunCC workaround. Some versions
42 // of SunCC lose/drop the const-ness in the F1 and F4 functions. It eventually
43 // results in a failed link due to the const/non-const mismatch.
44 
45 #ifndef CRYPTOPP_ADVANCED_SIMD_TEMPLATES
46 #define CRYPTOPP_ADVANCED_SIMD_TEMPLATES
47 
48 #include "config.h"
49 #include "misc.h"
50 #include "stdcpp.h"
51 
52 #if (CRYPTOPP_ARM_NEON_AVAILABLE)
53 # include <arm_neon.h>
54 #endif
55 
56 #if (CRYPTOPP_ARM_ACLE_AVAILABLE)
57 # include <stdint.h>
58 # include <arm_acle.h>
59 #endif
60 
61 #if (CRYPTOPP_SSE2_INTRIN_AVAILABLE)
62 # include <emmintrin.h>
63 # include <xmmintrin.h>
64 #endif
65 
66 // SunCC needs CRYPTOPP_SSSE3_AVAILABLE, too
67 #if (CRYPTOPP_SSSE3_AVAILABLE)
68 # include <emmintrin.h>
69 # include <pmmintrin.h>
70 # include <xmmintrin.h>
71 #endif
72 
73 #if defined(__ALTIVEC__)
74 # include "ppc_simd.h"
75 #endif
76 
77 #ifndef CRYPTOPP_INLINE
78 # if defined(CRYPTOPP_DEBUG)
79 # define CRYPTOPP_INLINE static
80 # else
81 # define CRYPTOPP_INLINE inline
82 # endif
83 #endif
84 
85 // ************************ All block ciphers *********************** //
86 
87 ANONYMOUS_NAMESPACE_BEGIN
88 
89 using CryptoPP::BlockTransformation;
90 
91 CRYPTOPP_CONSTANT(BT_XorInput = BlockTransformation::BT_XorInput)
92 CRYPTOPP_CONSTANT(BT_AllowParallel = BlockTransformation::BT_AllowParallel)
93 CRYPTOPP_CONSTANT(BT_InBlockIsCounter = BlockTransformation::BT_InBlockIsCounter)
94 CRYPTOPP_CONSTANT(BT_ReverseDirection = BlockTransformation::BT_ReverseDirection)
95 CRYPTOPP_CONSTANT(BT_DontIncrementInOutPointers = BlockTransformation::BT_DontIncrementInOutPointers)
96 
97 ANONYMOUS_NAMESPACE_END
98 
99 // *************************** ARM NEON ************************** //
100 
101 #if (CRYPTOPP_ARM_NEON_AVAILABLE)
102 
103 NAMESPACE_BEGIN(CryptoPP)
104 
105 /// \brief AdvancedProcessBlocks for 2 and 6 blocks
106 /// \tparam F2 function to process 2 64-bit blocks
107 /// \tparam F6 function to process 6 64-bit blocks
108 /// \tparam W word type of the subkey table
109 /// \details AdvancedProcessBlocks64_6x2_NEON processes 6 and 2 NEON SIMD words
110 /// at a time. For a single block the template uses F2 with a zero block.
111 /// \details The subkey type is usually word32 or word64. F2 and F6 must use the
112 /// same word type.
113 template <typename F2, typename F6, typename W>
114 CRYPTOPP_INLINE size_t AdvancedProcessBlocks64_6x2_NEON(F2 func2, F6 func6,
115  const W *subKeys, size_t rounds, const byte *inBlocks,
116  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
117 {
118  CRYPTOPP_ASSERT(subKeys);
119  CRYPTOPP_ASSERT(inBlocks);
120  CRYPTOPP_ASSERT(outBlocks);
121  CRYPTOPP_ASSERT(length >= 8);
122 
123 #if (CRYPTOPP_LITTLE_ENDIAN)
124  const uint32x4_t s_one = {0, 0, 0, 1<<24};
125  const uint32x4_t s_two = {0, 2<<24, 0, 2<<24};
126 #else
127  // TODO: verify these constants on ARM-BE
128  const uint32x4_t s_one = {0, 0, 0, 1};
129  const uint32x4_t s_two = {0, 2, 0, 2};
130 #endif
131 
132  const size_t blockSize = 8;
133  const size_t neonBlockSize = 16;
134 
135  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : neonBlockSize;
136  size_t xorIncrement = (xorBlocks != NULLPTR) ? neonBlockSize : 0;
137  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : neonBlockSize;
138 
139  // Clang and Coverity are generating findings using xorBlocks as a flag.
140  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
141  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
142 
143  if (flags & BT_ReverseDirection)
144  {
145  inBlocks = PtrAdd(inBlocks, length - neonBlockSize);
146  xorBlocks = PtrAdd(xorBlocks, length - neonBlockSize);
147  outBlocks = PtrAdd(outBlocks, length - neonBlockSize);
148  inIncrement = 0-inIncrement;
149  xorIncrement = 0-xorIncrement;
150  outIncrement = 0-outIncrement;
151  }
152 
153  if (flags & BT_AllowParallel)
154  {
155  while (length >= 6*neonBlockSize)
156  {
157  uint32x4_t block0, block1, block2, block3, block4, block5;
158  if (flags & BT_InBlockIsCounter)
159  {
160  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
161  // After the dup load we have two counters in the NEON word. Then we need
162  // to increment the low ctr by 0 and the high ctr by 1.
163  const uint8x8_t ctr = vld1_u8(inBlocks);
164  block0 = vaddq_u32(s_one, vreinterpretq_u32_u8(vcombine_u8(ctr,ctr)));
165 
166  // After initial increment of {0,1} remaining counters increment by {2,2}.
167  block1 = vaddq_u32(s_two, block0);
168  block2 = vaddq_u32(s_two, block1);
169  block3 = vaddq_u32(s_two, block2);
170  block4 = vaddq_u32(s_two, block3);
171  block5 = vaddq_u32(s_two, block4);
172 
173  vst1_u8(const_cast<byte*>(inBlocks), vget_low_u8(
174  vreinterpretq_u8_u32(vaddq_u32(s_two, block5))));
175  }
176  else
177  {
178  block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
179  inBlocks = PtrAdd(inBlocks, inIncrement);
180  block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
181  inBlocks = PtrAdd(inBlocks, inIncrement);
182  block2 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
183  inBlocks = PtrAdd(inBlocks, inIncrement);
184  block3 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
185  inBlocks = PtrAdd(inBlocks, inIncrement);
186  block4 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
187  inBlocks = PtrAdd(inBlocks, inIncrement);
188  block5 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
189  inBlocks = PtrAdd(inBlocks, inIncrement);
190  }
191 
192  if (xorInput)
193  {
194  block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
195  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
196  block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
197  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
198  block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
199  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
200  block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
201  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
202  block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
203  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
204  block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
205  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
206  }
207 
208  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
209 
210  if (xorOutput)
211  {
212  block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
213  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
214  block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
215  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
216  block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
217  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
218  block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
219  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
220  block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
221  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
222  block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
223  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
224  }
225 
226  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block0));
227  outBlocks = PtrAdd(outBlocks, outIncrement);
228  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block1));
229  outBlocks = PtrAdd(outBlocks, outIncrement);
230  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block2));
231  outBlocks = PtrAdd(outBlocks, outIncrement);
232  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block3));
233  outBlocks = PtrAdd(outBlocks, outIncrement);
234  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block4));
235  outBlocks = PtrAdd(outBlocks, outIncrement);
236  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block5));
237  outBlocks = PtrAdd(outBlocks, outIncrement);
238 
239  length -= 6*neonBlockSize;
240  }
241 
242  while (length >= 2*neonBlockSize)
243  {
244  uint32x4_t block0, block1;
245  if (flags & BT_InBlockIsCounter)
246  {
247  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
248  // After the dup load we have two counters in the NEON word. Then we need
249  // to increment the low ctr by 0 and the high ctr by 1.
250  const uint8x8_t ctr = vld1_u8(inBlocks);
251  block0 = vaddq_u32(s_one, vreinterpretq_u32_u8(vcombine_u8(ctr,ctr)));
252 
253  // After initial increment of {0,1} remaining counters increment by {2,2}.
254  block1 = vaddq_u32(s_two, block0);
255 
256  vst1_u8(const_cast<byte*>(inBlocks), vget_low_u8(
257  vreinterpretq_u8_u32(vaddq_u32(s_two, block1))));
258  }
259  else
260  {
261  block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
262  inBlocks = PtrAdd(inBlocks, inIncrement);
263  block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
264  inBlocks = PtrAdd(inBlocks, inIncrement);
265  }
266 
267  if (xorInput)
268  {
269  block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
270  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
271  block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
272  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
273  }
274 
275  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
276 
277  if (xorOutput)
278  {
279  block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
280  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
281  block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
282  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
283  }
284 
285  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block0));
286  outBlocks = PtrAdd(outBlocks, outIncrement);
287  vst1q_u8(outBlocks, vreinterpretq_u8_u32(block1));
288  outBlocks = PtrAdd(outBlocks, outIncrement);
289 
290  length -= 2*neonBlockSize;
291  }
292  }
293 
294  if (length)
295  {
296  // Adjust to real block size
297  if (flags & BT_ReverseDirection)
298  {
299  inIncrement += inIncrement ? blockSize : 0;
300  xorIncrement += xorIncrement ? blockSize : 0;
301  outIncrement += outIncrement ? blockSize : 0;
302  inBlocks = PtrSub(inBlocks, inIncrement);
303  xorBlocks = PtrSub(xorBlocks, xorIncrement);
304  outBlocks = PtrSub(outBlocks, outIncrement);
305  }
306  else
307  {
308  inIncrement -= inIncrement ? blockSize : 0;
309  xorIncrement -= xorIncrement ? blockSize : 0;
310  outIncrement -= outIncrement ? blockSize : 0;
311  }
312 
313  while (length >= blockSize)
314  {
315  uint32x4_t block, zero = {0};
316 
317  const uint8x8_t v = vld1_u8(inBlocks);
318  block = vreinterpretq_u32_u8(vcombine_u8(v,v));
319 
320  if (xorInput)
321  {
322  const uint8x8_t x = vld1_u8(xorBlocks);
323  block = veorq_u32(block, vreinterpretq_u32_u8(vcombine_u8(x,x)));
324  }
325 
326  if (flags & BT_InBlockIsCounter)
327  const_cast<byte *>(inBlocks)[7]++;
328 
329  func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
330 
331  if (xorOutput)
332  {
333  const uint8x8_t x = vld1_u8(xorBlocks);
334  block = veorq_u32(block, vreinterpretq_u32_u8(vcombine_u8(x,x)));
335  }
336 
337  vst1_u8(const_cast<byte*>(outBlocks),
338  vget_low_u8(vreinterpretq_u8_u32(block)));
339 
340  inBlocks = PtrAdd(inBlocks, inIncrement);
341  outBlocks = PtrAdd(outBlocks, outIncrement);
342  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
343  length -= blockSize;
344  }
345  }
346 
347  return length;
348 }
349 
350 /// \brief AdvancedProcessBlocks for 1 and 6 blocks
351 /// \tparam F1 function to process 1 128-bit block
352 /// \tparam F6 function to process 6 128-bit blocks
353 /// \tparam W word type of the subkey table
354 /// \details AdvancedProcessBlocks128_6x1_NEON processes 6 and 2 NEON SIMD words
355 /// at a time.
356 /// \details The subkey type is usually word32 or word64. F1 and F6 must use the
357 /// same word type.
358 template <typename F1, typename F6, typename W>
359 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_6x1_NEON(F1 func1, F6 func6,
360  const W *subKeys, size_t rounds, const byte *inBlocks,
361  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
362 {
363  CRYPTOPP_ASSERT(subKeys);
364  CRYPTOPP_ASSERT(inBlocks);
365  CRYPTOPP_ASSERT(outBlocks);
366  CRYPTOPP_ASSERT(length >= 16);
367 
368 #if (CRYPTOPP_LITTLE_ENDIAN)
369  const uint32x4_t s_one = {0, 0, 0, 1<<24};
370  //const uint32x4_t s_two = {0, 2<<24, 0, 2<<24};
371 #else
372  // TODO: verify these constants on ARM-BE
373  const uint32x4_t s_one = {0, 0, 0, 1};
374  //const uint32x4_t s_two = {0, 2, 0, 2};
375 #endif
376 
377  const size_t blockSize = 16;
378  // const size_t neonBlockSize = 16;
379 
380  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
381  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
382  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
383 
384  // Clang and Coverity are generating findings using xorBlocks as a flag.
385  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
386  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
387 
388  if (flags & BT_ReverseDirection)
389  {
390  inBlocks = PtrAdd(inBlocks, length - blockSize);
391  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
392  outBlocks = PtrAdd(outBlocks, length - blockSize);
393  inIncrement = 0-inIncrement;
394  xorIncrement = 0-xorIncrement;
395  outIncrement = 0-outIncrement;
396  }
397 
398  if (flags & BT_AllowParallel)
399  {
400  while (length >= 6*blockSize)
401  {
402  uint64x2_t block0, block1, block2, block3, block4, block5;
403  if (flags & BT_InBlockIsCounter)
404  {
405  const uint64x2_t one = vreinterpretq_u64_u32(s_one);
406  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
407  block1 = vaddq_u64(block0, one);
408  block2 = vaddq_u64(block1, one);
409  block3 = vaddq_u64(block2, one);
410  block4 = vaddq_u64(block3, one);
411  block5 = vaddq_u64(block4, one);
412  vst1q_u8(const_cast<byte*>(inBlocks),
413  vreinterpretq_u8_u64(vaddq_u64(block5, one)));
414  }
415  else
416  {
417  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
418  inBlocks = PtrAdd(inBlocks, inIncrement);
419  block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
420  inBlocks = PtrAdd(inBlocks, inIncrement);
421  block2 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
422  inBlocks = PtrAdd(inBlocks, inIncrement);
423  block3 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
424  inBlocks = PtrAdd(inBlocks, inIncrement);
425  block4 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
426  inBlocks = PtrAdd(inBlocks, inIncrement);
427  block5 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
428  inBlocks = PtrAdd(inBlocks, inIncrement);
429  }
430 
431  if (xorInput)
432  {
433  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
434  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
435  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
436  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
437  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
438  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
439  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
440  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
441  block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
442  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
443  block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
444  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
445  }
446 
447  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
448 
449  if (xorOutput)
450  {
451  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
452  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
453  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
454  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
455  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
456  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
457  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
458  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
459  block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
460  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
461  block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
462  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
463  }
464 
465  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
466  outBlocks = PtrAdd(outBlocks, outIncrement);
467  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
468  outBlocks = PtrAdd(outBlocks, outIncrement);
469  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block2));
470  outBlocks = PtrAdd(outBlocks, outIncrement);
471  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block3));
472  outBlocks = PtrAdd(outBlocks, outIncrement);
473  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block4));
474  outBlocks = PtrAdd(outBlocks, outIncrement);
475  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block5));
476  outBlocks = PtrAdd(outBlocks, outIncrement);
477 
478  length -= 6*blockSize;
479  }
480  }
481 
482  while (length >= blockSize)
483  {
484  uint64x2_t block;
485  block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
486 
487  if (xorInput)
488  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
489 
490  if (flags & BT_InBlockIsCounter)
491  const_cast<byte *>(inBlocks)[15]++;
492 
493  func1(block, subKeys, static_cast<unsigned int>(rounds));
494 
495  if (xorOutput)
496  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
497 
498  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block));
499 
500  inBlocks = PtrAdd(inBlocks, inIncrement);
501  outBlocks = PtrAdd(outBlocks, outIncrement);
502  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
503  length -= blockSize;
504  }
505 
506  return length;
507 }
508 
509 /// \brief AdvancedProcessBlocks for 1 and 4 blocks
510 /// \tparam F1 function to process 1 128-bit block
511 /// \tparam F4 function to process 4 128-bit blocks
512 /// \tparam W word type of the subkey table
513 /// \tparam V vector type of the NEON datatype
514 /// \details AdvancedProcessBlocks128_4x1_NEON processes 4 and 1 NEON SIMD words
515 /// at a time.
516 /// \details The subkey type is usually word32 or word64. V is the vector type and it is
517 /// usually uint32x4_t or uint64x2_t. F1, F4, W and V must use the same word and
518 /// vector type. The V parameter is used to avoid template argument
519 /// deduction/substitution failures.
520 template <typename F1, typename F4, typename W, typename V>
521 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_4x1_NEON(F1 func1, F4 func4,
522  const V& unused, const W *subKeys, size_t rounds, const byte *inBlocks,
523  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
524 {
525  CRYPTOPP_ASSERT(subKeys);
526  CRYPTOPP_ASSERT(inBlocks);
527  CRYPTOPP_ASSERT(outBlocks);
528  CRYPTOPP_ASSERT(length >= 16);
529  CRYPTOPP_UNUSED(unused);
530 
531 #if (CRYPTOPP_LITTLE_ENDIAN)
532  const uint32x4_t s_one = {0, 0, 0, 1<<24};
533  //const uint32x4_t s_two = {0, 2<<24, 0, 2<<24};
534 #else
535  // TODO: verify these constants on ARM-BE
536  const uint32x4_t s_one = {0, 0, 0, 1};
537  //const uint32x4_t s_two = {0, 2, 0, 2};
538 #endif
539 
540  const size_t blockSize = 16;
541  // const size_t neonBlockSize = 16;
542 
543  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
544  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
545  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
546 
547  // Clang and Coverity are generating findings using xorBlocks as a flag.
548  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
549  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
550 
551  if (flags & BT_ReverseDirection)
552  {
553  inBlocks = PtrAdd(inBlocks, length - blockSize);
554  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
555  outBlocks = PtrAdd(outBlocks, length - blockSize);
556  inIncrement = 0-inIncrement;
557  xorIncrement = 0-xorIncrement;
558  outIncrement = 0-outIncrement;
559  }
560 
561  if (flags & BT_AllowParallel)
562  {
563  while (length >= 4*blockSize)
564  {
565  uint64x2_t block0, block1, block2, block3;
566  if (flags & BT_InBlockIsCounter)
567  {
568  const uint64x2_t one = vreinterpretq_u64_u32(s_one);
569  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
570  block1 = vaddq_u64(block0, one);
571  block2 = vaddq_u64(block1, one);
572  block3 = vaddq_u64(block2, one);
573  vst1q_u8(const_cast<byte*>(inBlocks),
574  vreinterpretq_u8_u64(vaddq_u64(block3, one)));
575  }
576  else
577  {
578  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
579  inBlocks = PtrAdd(inBlocks, inIncrement);
580  block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
581  inBlocks = PtrAdd(inBlocks, inIncrement);
582  block2 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
583  inBlocks = PtrAdd(inBlocks, inIncrement);
584  block3 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
585  inBlocks = PtrAdd(inBlocks, inIncrement);
586  }
587 
588  if (xorInput)
589  {
590  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
591  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
592  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
593  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
594  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
595  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
596  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
597  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
598  }
599 
600  func4((V&)block0, (V&)block1, (V&)block2, (V&)block3, subKeys, static_cast<unsigned int>(rounds));
601 
602  if (xorOutput)
603  {
604  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
605  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
606  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
607  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
608  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
609  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
610  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
611  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
612  }
613 
614  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
615  outBlocks = PtrAdd(outBlocks, outIncrement);
616  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
617  outBlocks = PtrAdd(outBlocks, outIncrement);
618  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block2));
619  outBlocks = PtrAdd(outBlocks, outIncrement);
620  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block3));
621  outBlocks = PtrAdd(outBlocks, outIncrement);
622 
623  length -= 4*blockSize;
624  }
625  }
626 
627  while (length >= blockSize)
628  {
629  uint64x2_t block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
630 
631  if (xorInput)
632  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
633 
634  if (flags & BT_InBlockIsCounter)
635  const_cast<byte *>(inBlocks)[15]++;
636 
637  func1( (V&)block, subKeys, static_cast<unsigned int>(rounds));
638 
639  if (xorOutput)
640  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
641 
642  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block));
643 
644  inBlocks = PtrAdd(inBlocks, inIncrement);
645  outBlocks = PtrAdd(outBlocks, outIncrement);
646  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
647  length -= blockSize;
648  }
649 
650  return length;
651 }
652 
653 /// \brief AdvancedProcessBlocks for 2 and 6 blocks
654 /// \tparam F2 function to process 2 128-bit blocks
655 /// \tparam F6 function to process 6 128-bit blocks
656 /// \tparam W word type of the subkey table
657 /// \details AdvancedProcessBlocks128_6x2_NEON processes 6 and 2 NEON SIMD words
658 /// at a time. For a single block the template uses F2 with a zero block.
659 /// \details The subkey type is usually word32 or word64. F2 and F6 must use the
660 /// same word type.
661 template <typename F2, typename F6, typename W>
662 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_6x2_NEON(F2 func2, F6 func6,
663  const W *subKeys, size_t rounds, const byte *inBlocks,
664  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
665 {
666  CRYPTOPP_ASSERT(subKeys);
667  CRYPTOPP_ASSERT(inBlocks);
668  CRYPTOPP_ASSERT(outBlocks);
669  CRYPTOPP_ASSERT(length >= 16);
670 
671 #if (CRYPTOPP_LITTLE_ENDIAN)
672  const uint32x4_t s_one = {0, 0, 0, 1<<24};
673  //const uint32x4_t s_two = {0, 2<<24, 0, 2<<24};
674 #else
675  // TODO: verify these constants on ARM-BE
676  const uint32x4_t s_one = {0, 0, 0, 1};
677  //const uint32x4_t s_two = {0, 2, 0, 2};
678 #endif
679 
680  const size_t blockSize = 16;
681  // const size_t neonBlockSize = 16;
682 
683  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
684  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
685  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
686 
687  // Clang and Coverity are generating findings using xorBlocks as a flag.
688  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
689  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
690 
691  if (flags & BT_ReverseDirection)
692  {
693  inBlocks = PtrAdd(inBlocks, length - blockSize);
694  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
695  outBlocks = PtrAdd(outBlocks, length - blockSize);
696  inIncrement = 0-inIncrement;
697  xorIncrement = 0-xorIncrement;
698  outIncrement = 0-outIncrement;
699  }
700 
701  if (flags & BT_AllowParallel)
702  {
703  while (length >= 6*blockSize)
704  {
705  uint64x2_t block0, block1, block2, block3, block4, block5;
706  if (flags & BT_InBlockIsCounter)
707  {
708  const uint64x2_t one = vreinterpretq_u64_u32(s_one);
709  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
710  block1 = vaddq_u64(block0, one);
711  block2 = vaddq_u64(block1, one);
712  block3 = vaddq_u64(block2, one);
713  block4 = vaddq_u64(block3, one);
714  block5 = vaddq_u64(block4, one);
715  vst1q_u8(const_cast<byte*>(inBlocks),
716  vreinterpretq_u8_u64(vaddq_u64(block5, one)));
717  }
718  else
719  {
720  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
721  inBlocks = PtrAdd(inBlocks, inIncrement);
722  block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
723  inBlocks = PtrAdd(inBlocks, inIncrement);
724  block2 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
725  inBlocks = PtrAdd(inBlocks, inIncrement);
726  block3 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
727  inBlocks = PtrAdd(inBlocks, inIncrement);
728  block4 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
729  inBlocks = PtrAdd(inBlocks, inIncrement);
730  block5 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
731  inBlocks = PtrAdd(inBlocks, inIncrement);
732  }
733 
734  if (xorInput)
735  {
736  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
737  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
738  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
739  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
740  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
741  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
742  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
743  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
744  block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
745  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
746  block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
747  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
748  }
749 
750  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
751 
752  if (xorOutput)
753  {
754  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
755  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
756  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
757  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
758  block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
759  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
760  block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
761  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
762  block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
763  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
764  block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
765  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
766  }
767 
768  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
769  outBlocks = PtrAdd(outBlocks, outIncrement);
770  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
771  outBlocks = PtrAdd(outBlocks, outIncrement);
772  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block2));
773  outBlocks = PtrAdd(outBlocks, outIncrement);
774  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block3));
775  outBlocks = PtrAdd(outBlocks, outIncrement);
776  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block4));
777  outBlocks = PtrAdd(outBlocks, outIncrement);
778  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block5));
779  outBlocks = PtrAdd(outBlocks, outIncrement);
780 
781  length -= 6*blockSize;
782  }
783 
784  while (length >= 2*blockSize)
785  {
786  uint64x2_t block0, block1;
787  if (flags & BT_InBlockIsCounter)
788  {
789  const uint64x2_t one = vreinterpretq_u64_u32(s_one);
790  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
791  block1 = vaddq_u64(block0, one);
792  vst1q_u8(const_cast<byte*>(inBlocks),
793  vreinterpretq_u8_u64(vaddq_u64(block1, one)));
794  }
795  else
796  {
797  block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
798  inBlocks = PtrAdd(inBlocks, inIncrement);
799  block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
800  inBlocks = PtrAdd(inBlocks, inIncrement);
801  }
802 
803  if (xorInput)
804  {
805  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
806  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
807  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
808  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
809  }
810 
811  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
812 
813  if (xorOutput)
814  {
815  block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
816  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
817  block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
818  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
819  }
820 
821  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
822  outBlocks = PtrAdd(outBlocks, outIncrement);
823  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
824  outBlocks = PtrAdd(outBlocks, outIncrement);
825 
826  length -= 2*blockSize;
827  }
828  }
829 
830  while (length >= blockSize)
831  {
832  uint64x2_t block, zero = {0,0};
833  block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
834 
835  if (xorInput)
836  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
837 
838  if (flags & BT_InBlockIsCounter)
839  const_cast<byte *>(inBlocks)[15]++;
840 
841  func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
842 
843  if (xorOutput)
844  block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
845 
846  vst1q_u8(outBlocks, vreinterpretq_u8_u64(block));
847 
848  inBlocks = PtrAdd(inBlocks, inIncrement);
849  outBlocks = PtrAdd(outBlocks, outIncrement);
850  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
851  length -= blockSize;
852  }
853 
854  return length;
855 }
856 
857 NAMESPACE_END // CryptoPP
858 
859 #endif // CRYPTOPP_ARM_NEON_AVAILABLE
860 
861 // *************************** Intel SSE ************************** //
862 
863 #if defined(CRYPTOPP_SSSE3_AVAILABLE)
864 
865 // Hack for SunCC, http://github.com/weidai11/cryptopp/issues/224
866 #if (__SUNPRO_CC >= 0x5130)
867 # define MAYBE_CONST
868 # define MAYBE_UNCONST_CAST(T, x) const_cast<MAYBE_CONST T>(x)
869 #else
870 # define MAYBE_CONST const
871 # define MAYBE_UNCONST_CAST(T, x) (x)
872 #endif
873 
874 // Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
875 #ifndef M128_CAST
876 # define M128_CAST(x) ((__m128i *)(void *)(x))
877 #endif
878 #ifndef CONST_M128_CAST
879 # define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
880 #endif
881 
882 NAMESPACE_BEGIN(CryptoPP)
883 
884 /// \brief AdvancedProcessBlocks for 1 and 2 blocks
885 /// \tparam F1 function to process 1 64-bit block
886 /// \tparam F2 function to process 2 64-bit blocks
887 /// \tparam W word type of the subkey table
888 /// \details AdvancedProcessBlocks64_2x1_SSE processes 2 and 1 SSE SIMD words
889 /// at a time.
890 /// \details The subkey type is usually word32 or word64. F1 and F2 must use the
891 /// same word type.
892 template <typename F1, typename F2, typename W>
893 CRYPTOPP_INLINE size_t AdvancedProcessBlocks64_2x1_SSE(F1 func1, F2 func2,
894  MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
895  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
896 {
897  CRYPTOPP_ASSERT(subKeys);
898  CRYPTOPP_ASSERT(inBlocks);
899  CRYPTOPP_ASSERT(outBlocks);
900  CRYPTOPP_ASSERT(length >= 8);
901 
902  const size_t blockSize = 8;
903  const size_t xmmBlockSize = 16;
904 
905  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : xmmBlockSize;
906  size_t xorIncrement = (xorBlocks != NULLPTR) ? xmmBlockSize : 0;
907  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : xmmBlockSize;
908 
909  // Clang and Coverity are generating findings using xorBlocks as a flag.
910  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
911  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
912 
913  if (flags & BT_ReverseDirection)
914  {
915  inBlocks = PtrAdd(inBlocks, length - xmmBlockSize);
916  xorBlocks = PtrAdd(xorBlocks, length - xmmBlockSize);
917  outBlocks = PtrAdd(outBlocks, length - xmmBlockSize);
918  inIncrement = 0-inIncrement;
919  xorIncrement = 0-xorIncrement;
920  outIncrement = 0-outIncrement;
921  }
922 
923  if (flags & BT_AllowParallel)
924  {
925  double temp[2];
926  while (length >= 2*xmmBlockSize)
927  {
928  __m128i block0, block1;
929  if (flags & BT_InBlockIsCounter)
930  {
931  // Increment of 1 and 2 in big-endian compatible with the ctr byte array.
932  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
933  const __m128i s_two = _mm_set_epi32(2<<24, 0, 2<<24, 0);
934 
935  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
936  // After the dup load we have two counters in the XMM word. Then we need
937  // to increment the low ctr by 0 and the high ctr by 1.
938  std::memcpy(temp, inBlocks, blockSize);
939  block0 = _mm_add_epi32(s_one, _mm_castpd_si128(_mm_loaddup_pd(temp)));
940 
941  // After initial increment of {0,1} remaining counters increment by {2,2}.
942  block1 = _mm_add_epi32(s_two, block0);
943 
944  // Store the next counter. When BT_InBlockIsCounter is set then
945  // inBlocks is backed by m_counterArray which is non-const.
946  _mm_store_sd(temp, _mm_castsi128_pd(_mm_add_epi64(s_two, block1)));
947  std::memcpy(const_cast<byte*>(inBlocks), temp, blockSize);
948  }
949  else
950  {
951  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
952  inBlocks = PtrAdd(inBlocks, inIncrement);
953  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
954  inBlocks = PtrAdd(inBlocks, inIncrement);
955  }
956 
957  if (xorInput)
958  {
959  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
960  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
961  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
962  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
963  }
964 
965  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
966 
967  if (xorOutput)
968  {
969  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
970  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
971  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
972  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
973  }
974 
975  _mm_storeu_si128(M128_CAST(outBlocks), block0);
976  outBlocks = PtrAdd(outBlocks, outIncrement);
977  _mm_storeu_si128(M128_CAST(outBlocks), block1);
978  outBlocks = PtrAdd(outBlocks, outIncrement);
979 
980  length -= 2*xmmBlockSize;
981  }
982  }
983 
984  if (length)
985  {
986  // Adjust to real block size
987  if (flags & BT_ReverseDirection)
988  {
989  inIncrement += inIncrement ? blockSize : 0;
990  xorIncrement += xorIncrement ? blockSize : 0;
991  outIncrement += outIncrement ? blockSize : 0;
992  inBlocks = PtrSub(inBlocks, inIncrement);
993  xorBlocks = PtrSub(xorBlocks, xorIncrement);
994  outBlocks = PtrSub(outBlocks, outIncrement);
995  }
996  else
997  {
998  inIncrement -= inIncrement ? blockSize : 0;
999  xorIncrement -= xorIncrement ? blockSize : 0;
1000  outIncrement -= outIncrement ? blockSize : 0;
1001  }
1002 
1003  while (length >= blockSize)
1004  {
1005  double temp[2];
1006  std::memcpy(temp, inBlocks, blockSize);
1007  __m128i block = _mm_castpd_si128(_mm_load_sd(temp));
1008 
1009  if (xorInput)
1010  {
1011  std::memcpy(temp, xorBlocks, blockSize);
1012  block = _mm_xor_si128(block, _mm_castpd_si128(_mm_load_sd(temp)));
1013  }
1014 
1015  if (flags & BT_InBlockIsCounter)
1016  const_cast<byte *>(inBlocks)[7]++;
1017 
1018  func1(block, subKeys, static_cast<unsigned int>(rounds));
1019 
1020  if (xorOutput)
1021  {
1022  std::memcpy(temp, xorBlocks, blockSize);
1023  block = _mm_xor_si128(block, _mm_castpd_si128(_mm_load_sd(temp)));
1024  }
1025 
1026  _mm_store_sd(temp, _mm_castsi128_pd(block));
1027  std::memcpy(outBlocks, temp, blockSize);
1028 
1029  inBlocks = PtrAdd(inBlocks, inIncrement);
1030  outBlocks = PtrAdd(outBlocks, outIncrement);
1031  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1032  length -= blockSize;
1033  }
1034  }
1035 
1036  return length;
1037 }
1038 
1039 /// \brief AdvancedProcessBlocks for 2 and 6 blocks
1040 /// \tparam F2 function to process 2 64-bit blocks
1041 /// \tparam F6 function to process 6 64-bit blocks
1042 /// \tparam W word type of the subkey table
1043 /// \details AdvancedProcessBlocks64_6x2_SSE processes 6 and 2 SSE SIMD words
1044 /// at a time. For a single block the template uses F2 with a zero block.
1045 /// \details The subkey type is usually word32 or word64. F2 and F6 must use the
1046 /// same word type.
1047 template <typename F2, typename F6, typename W>
1048 CRYPTOPP_INLINE size_t AdvancedProcessBlocks64_6x2_SSE(F2 func2, F6 func6,
1049  MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
1050  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
1051 {
1052  CRYPTOPP_ASSERT(subKeys);
1053  CRYPTOPP_ASSERT(inBlocks);
1054  CRYPTOPP_ASSERT(outBlocks);
1055  CRYPTOPP_ASSERT(length >= 8);
1056 
1057  const size_t blockSize = 8;
1058  const size_t xmmBlockSize = 16;
1059 
1060  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : xmmBlockSize;
1061  size_t xorIncrement = (xorBlocks != NULLPTR) ? xmmBlockSize : 0;
1062  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : xmmBlockSize;
1063 
1064  // Clang and Coverity are generating findings using xorBlocks as a flag.
1065  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
1066  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
1067 
1068  if (flags & BT_ReverseDirection)
1069  {
1070  inBlocks = PtrAdd(inBlocks, length - xmmBlockSize);
1071  xorBlocks = PtrAdd(xorBlocks, length - xmmBlockSize);
1072  outBlocks = PtrAdd(outBlocks, length - xmmBlockSize);
1073  inIncrement = 0-inIncrement;
1074  xorIncrement = 0-xorIncrement;
1075  outIncrement = 0-outIncrement;
1076  }
1077 
1078  if (flags & BT_AllowParallel)
1079  {
1080  double temp[2];
1081  while (length >= 6*xmmBlockSize)
1082  {
1083  __m128i block0, block1, block2, block3, block4, block5;
1084  if (flags & BT_InBlockIsCounter)
1085  {
1086  // Increment of 1 and 2 in big-endian compatible with the ctr byte array.
1087  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1088  const __m128i s_two = _mm_set_epi32(2<<24, 0, 2<<24, 0);
1089 
1090  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
1091  // After the dup load we have two counters in the XMM word. Then we need
1092  // to increment the low ctr by 0 and the high ctr by 1.
1093  std::memcpy(temp, inBlocks, blockSize);
1094  block0 = _mm_add_epi32(s_one, _mm_castpd_si128(_mm_loaddup_pd(temp)));
1095 
1096  // After initial increment of {0,1} remaining counters increment by {2,2}.
1097  block1 = _mm_add_epi32(s_two, block0);
1098  block2 = _mm_add_epi32(s_two, block1);
1099  block3 = _mm_add_epi32(s_two, block2);
1100  block4 = _mm_add_epi32(s_two, block3);
1101  block5 = _mm_add_epi32(s_two, block4);
1102 
1103  // Store the next counter. When BT_InBlockIsCounter is set then
1104  // inBlocks is backed by m_counterArray which is non-const.
1105  _mm_store_sd(temp, _mm_castsi128_pd(_mm_add_epi32(s_two, block5)));
1106  std::memcpy(const_cast<byte*>(inBlocks), temp, blockSize);
1107  }
1108  else
1109  {
1110  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1111  inBlocks = PtrAdd(inBlocks, inIncrement);
1112  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1113  inBlocks = PtrAdd(inBlocks, inIncrement);
1114  block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1115  inBlocks = PtrAdd(inBlocks, inIncrement);
1116  block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1117  inBlocks = PtrAdd(inBlocks, inIncrement);
1118  block4 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1119  inBlocks = PtrAdd(inBlocks, inIncrement);
1120  block5 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1121  inBlocks = PtrAdd(inBlocks, inIncrement);
1122  }
1123 
1124  if (xorInput)
1125  {
1126  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1127  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1128  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1129  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1130  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1131  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1132  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1133  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1134  block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1135  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1136  block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1137  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1138  }
1139 
1140  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
1141 
1142  if (xorOutput)
1143  {
1144  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1145  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1146  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1147  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1148  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1149  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1150  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1151  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1152  block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1153  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1154  block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1155  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1156  }
1157 
1158  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1159  outBlocks = PtrAdd(outBlocks, outIncrement);
1160  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1161  outBlocks = PtrAdd(outBlocks, outIncrement);
1162  _mm_storeu_si128(M128_CAST(outBlocks), block2);
1163  outBlocks = PtrAdd(outBlocks, outIncrement);
1164  _mm_storeu_si128(M128_CAST(outBlocks), block3);
1165  outBlocks = PtrAdd(outBlocks, outIncrement);
1166  _mm_storeu_si128(M128_CAST(outBlocks), block4);
1167  outBlocks = PtrAdd(outBlocks, outIncrement);
1168  _mm_storeu_si128(M128_CAST(outBlocks), block5);
1169  outBlocks = PtrAdd(outBlocks, outIncrement);
1170 
1171  length -= 6*xmmBlockSize;
1172  }
1173 
1174  while (length >= 2*xmmBlockSize)
1175  {
1176  __m128i block0, block1;
1177  if (flags & BT_InBlockIsCounter)
1178  {
1179  // Increment of 1 and 2 in big-endian compatible with the ctr byte array.
1180  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1181  const __m128i s_two = _mm_set_epi32(2<<24, 0, 2<<24, 0);
1182 
1183  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
1184  // After the dup load we have two counters in the XMM word. Then we need
1185  // to increment the low ctr by 0 and the high ctr by 1.
1186  std::memcpy(temp, inBlocks, blockSize);
1187  block0 = _mm_add_epi32(s_one, _mm_castpd_si128(_mm_loaddup_pd(temp)));
1188 
1189  // After initial increment of {0,1} remaining counters increment by {2,2}.
1190  block1 = _mm_add_epi32(s_two, block0);
1191 
1192  // Store the next counter. When BT_InBlockIsCounter is set then
1193  // inBlocks is backed by m_counterArray which is non-const.
1194  _mm_store_sd(temp, _mm_castsi128_pd(_mm_add_epi64(s_two, block1)));
1195  std::memcpy(const_cast<byte*>(inBlocks), temp, blockSize);
1196  }
1197  else
1198  {
1199  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1200  inBlocks = PtrAdd(inBlocks, inIncrement);
1201  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1202  inBlocks = PtrAdd(inBlocks, inIncrement);
1203  }
1204 
1205  if (xorInput)
1206  {
1207  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1208  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1209  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1210  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1211  }
1212 
1213  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
1214 
1215  if (xorOutput)
1216  {
1217  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1218  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1219  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1220  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1221  }
1222 
1223  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1224  outBlocks = PtrAdd(outBlocks, outIncrement);
1225  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1226  outBlocks = PtrAdd(outBlocks, outIncrement);
1227 
1228  length -= 2*xmmBlockSize;
1229  }
1230  }
1231 
1232  if (length)
1233  {
1234  // Adjust to real block size
1235  if (flags & BT_ReverseDirection)
1236  {
1237  inIncrement += inIncrement ? blockSize : 0;
1238  xorIncrement += xorIncrement ? blockSize : 0;
1239  outIncrement += outIncrement ? blockSize : 0;
1240  inBlocks = PtrSub(inBlocks, inIncrement);
1241  xorBlocks = PtrSub(xorBlocks, xorIncrement);
1242  outBlocks = PtrSub(outBlocks, outIncrement);
1243  }
1244  else
1245  {
1246  inIncrement -= inIncrement ? blockSize : 0;
1247  xorIncrement -= xorIncrement ? blockSize : 0;
1248  outIncrement -= outIncrement ? blockSize : 0;
1249  }
1250 
1251  while (length >= blockSize)
1252  {
1253  double temp[2];
1254  __m128i block, zero = _mm_setzero_si128();
1255  std::memcpy(temp, inBlocks, blockSize);
1256  block = _mm_castpd_si128(_mm_load_sd(temp));
1257 
1258  if (xorInput)
1259  {
1260  std::memcpy(temp, xorBlocks, blockSize);
1261  block = _mm_xor_si128(block,
1262  _mm_castpd_si128(_mm_load_sd(temp)));
1263  }
1264 
1265  if (flags & BT_InBlockIsCounter)
1266  const_cast<byte *>(inBlocks)[7]++;
1267 
1268  func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
1269 
1270  if (xorOutput)
1271  {
1272  std::memcpy(temp, xorBlocks, blockSize);
1273  block = _mm_xor_si128(block,
1274  _mm_castpd_si128(_mm_load_sd(temp)));
1275  }
1276 
1277  _mm_store_sd(temp, _mm_castsi128_pd(block));
1278  std::memcpy(outBlocks, temp, blockSize);
1279 
1280  inBlocks = PtrAdd(inBlocks, inIncrement);
1281  outBlocks = PtrAdd(outBlocks, outIncrement);
1282  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1283  length -= blockSize;
1284  }
1285  }
1286 
1287  return length;
1288 }
1289 
1290 /// \brief AdvancedProcessBlocks for 2 and 6 blocks
1291 /// \tparam F2 function to process 2 128-bit blocks
1292 /// \tparam F6 function to process 6 128-bit blocks
1293 /// \tparam W word type of the subkey table
1294 /// \details AdvancedProcessBlocks128_6x2_SSE processes 6 and 2 SSE SIMD words
1295 /// at a time. For a single block the template uses F2 with a zero block.
1296 /// \details The subkey type is usually word32 or word64. F2 and F6 must use the
1297 /// same word type.
1298 template <typename F2, typename F6, typename W>
1299 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_6x2_SSE(F2 func2, F6 func6,
1300  MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
1301  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
1302 {
1303  CRYPTOPP_ASSERT(subKeys);
1304  CRYPTOPP_ASSERT(inBlocks);
1305  CRYPTOPP_ASSERT(outBlocks);
1306  CRYPTOPP_ASSERT(length >= 16);
1307 
1308  const size_t blockSize = 16;
1309  // const size_t xmmBlockSize = 16;
1310 
1311  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
1312  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
1313  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
1314 
1315  // Clang and Coverity are generating findings using xorBlocks as a flag.
1316  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
1317  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
1318 
1319  if (flags & BT_ReverseDirection)
1320  {
1321  inBlocks = PtrAdd(inBlocks, length - blockSize);
1322  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
1323  outBlocks = PtrAdd(outBlocks, length - blockSize);
1324  inIncrement = 0-inIncrement;
1325  xorIncrement = 0-xorIncrement;
1326  outIncrement = 0-outIncrement;
1327  }
1328 
1329  if (flags & BT_AllowParallel)
1330  {
1331  while (length >= 6*blockSize)
1332  {
1333  __m128i block0, block1, block2, block3, block4, block5;
1334  if (flags & BT_InBlockIsCounter)
1335  {
1336  // Increment of 1 in big-endian compatible with the ctr byte array.
1337  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1338  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1339  block1 = _mm_add_epi32(block0, s_one);
1340  block2 = _mm_add_epi32(block1, s_one);
1341  block3 = _mm_add_epi32(block2, s_one);
1342  block4 = _mm_add_epi32(block3, s_one);
1343  block5 = _mm_add_epi32(block4, s_one);
1344  _mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block5, s_one));
1345  }
1346  else
1347  {
1348  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1349  inBlocks = PtrAdd(inBlocks, inIncrement);
1350  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1351  inBlocks = PtrAdd(inBlocks, inIncrement);
1352  block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1353  inBlocks = PtrAdd(inBlocks, inIncrement);
1354  block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1355  inBlocks = PtrAdd(inBlocks, inIncrement);
1356  block4 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1357  inBlocks = PtrAdd(inBlocks, inIncrement);
1358  block5 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1359  inBlocks = PtrAdd(inBlocks, inIncrement);
1360  }
1361 
1362  if (xorInput)
1363  {
1364  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1365  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1366  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1367  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1368  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1369  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1370  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1371  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1372  block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1373  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1374  block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1375  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1376  }
1377 
1378  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
1379 
1380  if (xorOutput)
1381  {
1382  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1383  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1384  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1385  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1386  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1387  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1388  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1389  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1390  block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1391  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1392  block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1393  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1394  }
1395 
1396  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1397  outBlocks = PtrAdd(outBlocks, outIncrement);
1398  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1399  outBlocks = PtrAdd(outBlocks, outIncrement);
1400  _mm_storeu_si128(M128_CAST(outBlocks), block2);
1401  outBlocks = PtrAdd(outBlocks, outIncrement);
1402  _mm_storeu_si128(M128_CAST(outBlocks), block3);
1403  outBlocks = PtrAdd(outBlocks, outIncrement);
1404  _mm_storeu_si128(M128_CAST(outBlocks), block4);
1405  outBlocks = PtrAdd(outBlocks, outIncrement);
1406  _mm_storeu_si128(M128_CAST(outBlocks), block5);
1407  outBlocks = PtrAdd(outBlocks, outIncrement);
1408 
1409  length -= 6*blockSize;
1410  }
1411 
1412  while (length >= 2*blockSize)
1413  {
1414  __m128i block0, block1;
1415  if (flags & BT_InBlockIsCounter)
1416  {
1417  // Increment of 1 in big-endian compatible with the ctr byte array.
1418  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1419  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1420  block1 = _mm_add_epi32(block0, s_one);
1421  _mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block1, s_one));
1422  }
1423  else
1424  {
1425  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1426  inBlocks = PtrAdd(inBlocks, inIncrement);
1427  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1428  inBlocks = PtrAdd(inBlocks, inIncrement);
1429  }
1430 
1431  if (xorInput)
1432  {
1433  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1434  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1435  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1436  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1437  }
1438 
1439  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
1440 
1441  if (xorOutput)
1442  {
1443  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1444  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1445  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1446  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1447  }
1448 
1449  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1450  outBlocks = PtrAdd(outBlocks, outIncrement);
1451  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1452  outBlocks = PtrAdd(outBlocks, outIncrement);
1453 
1454  length -= 2*blockSize;
1455  }
1456  }
1457 
1458  while (length >= blockSize)
1459  {
1460  __m128i block, zero = _mm_setzero_si128();
1461  block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1462 
1463  if (xorInput)
1464  block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1465 
1466  if (flags & BT_InBlockIsCounter)
1467  const_cast<byte *>(inBlocks)[15]++;
1468 
1469  func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
1470 
1471  if (xorOutput)
1472  block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1473 
1474  _mm_storeu_si128(M128_CAST(outBlocks), block);
1475 
1476  inBlocks = PtrAdd(inBlocks, inIncrement);
1477  outBlocks = PtrAdd(outBlocks, outIncrement);
1478  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1479  length -= blockSize;
1480  }
1481 
1482  return length;
1483 }
1484 
1485 /// \brief AdvancedProcessBlocks for 1 and 4 blocks
1486 /// \tparam F1 function to process 1 128-bit block
1487 /// \tparam F4 function to process 4 128-bit blocks
1488 /// \tparam W word type of the subkey table
1489 /// \details AdvancedProcessBlocks128_4x1_SSE processes 4 and 1 SSE SIMD words
1490 /// at a time.
1491 /// \details The subkey type is usually word32 or word64. F1 and F4 must use the
1492 /// same word type.
1493 template <typename F1, typename F4, typename W>
1494 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_4x1_SSE(F1 func1, F4 func4,
1495  MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
1496  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
1497 {
1498  CRYPTOPP_ASSERT(subKeys);
1499  CRYPTOPP_ASSERT(inBlocks);
1500  CRYPTOPP_ASSERT(outBlocks);
1501  CRYPTOPP_ASSERT(length >= 16);
1502 
1503  const size_t blockSize = 16;
1504  // const size_t xmmBlockSize = 16;
1505 
1506  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
1507  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
1508  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
1509 
1510  // Clang and Coverity are generating findings using xorBlocks as a flag.
1511  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
1512  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
1513 
1514  if (flags & BT_ReverseDirection)
1515  {
1516  inBlocks = PtrAdd(inBlocks, length - blockSize);
1517  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
1518  outBlocks = PtrAdd(outBlocks, length - blockSize);
1519  inIncrement = 0-inIncrement;
1520  xorIncrement = 0-xorIncrement;
1521  outIncrement = 0-outIncrement;
1522  }
1523 
1524  if (flags & BT_AllowParallel)
1525  {
1526  while (length >= 4*blockSize)
1527  {
1528  __m128i block0, block1, block2, block3;
1529  if (flags & BT_InBlockIsCounter)
1530  {
1531  // Increment of 1 in big-endian compatible with the ctr byte array.
1532  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1533  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1534  block1 = _mm_add_epi32(block0, s_one);
1535  block2 = _mm_add_epi32(block1, s_one);
1536  block3 = _mm_add_epi32(block2, s_one);
1537  _mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block3, s_one));
1538  }
1539  else
1540  {
1541  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1542  inBlocks = PtrAdd(inBlocks, inIncrement);
1543  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1544  inBlocks = PtrAdd(inBlocks, inIncrement);
1545  block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1546  inBlocks = PtrAdd(inBlocks, inIncrement);
1547  block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1548  inBlocks = PtrAdd(inBlocks, inIncrement);
1549  }
1550 
1551  if (xorInput)
1552  {
1553  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1554  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1555  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1556  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1557  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1558  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1559  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1560  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1561  }
1562 
1563  func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
1564 
1565  if (xorOutput)
1566  {
1567  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1568  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1569  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1570  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1571  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1572  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1573  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1574  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1575  }
1576 
1577  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1578  outBlocks = PtrAdd(outBlocks, outIncrement);
1579  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1580  outBlocks = PtrAdd(outBlocks, outIncrement);
1581  _mm_storeu_si128(M128_CAST(outBlocks), block2);
1582  outBlocks = PtrAdd(outBlocks, outIncrement);
1583  _mm_storeu_si128(M128_CAST(outBlocks), block3);
1584  outBlocks = PtrAdd(outBlocks, outIncrement);
1585 
1586  length -= 4*blockSize;
1587  }
1588  }
1589 
1590  while (length >= blockSize)
1591  {
1592  __m128i block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1593 
1594  if (xorInput)
1595  block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1596 
1597  if (flags & BT_InBlockIsCounter)
1598  const_cast<byte *>(inBlocks)[15]++;
1599 
1600  func1(block, subKeys, static_cast<unsigned int>(rounds));
1601 
1602  if (xorOutput)
1603  block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1604 
1605  _mm_storeu_si128(M128_CAST(outBlocks), block);
1606 
1607  inBlocks = PtrAdd(inBlocks, inIncrement);
1608  outBlocks = PtrAdd(outBlocks, outIncrement);
1609  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1610  length -= blockSize;
1611  }
1612 
1613  return length;
1614 }
1615 
1616 /// \brief AdvancedProcessBlocks for 1 and 4 blocks
1617 /// \tparam F1 function to process 1 64-bit block
1618 /// \tparam F4 function to process 6 64-bit blocks
1619 /// \tparam W word type of the subkey table
1620 /// \details AdvancedProcessBlocks64_4x1_SSE processes 4 and 1 SSE SIMD words
1621 /// at a time.
1622 /// \details The subkey type is usually word32 or word64. F1 and F4 must use the
1623 /// same word type.
1624 template <typename F1, typename F4, typename W>
1625 CRYPTOPP_INLINE size_t AdvancedProcessBlocks64_4x1_SSE(F1 func1, F4 func4,
1626  MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
1627  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
1628 {
1629  CRYPTOPP_ASSERT(subKeys);
1630  CRYPTOPP_ASSERT(inBlocks);
1631  CRYPTOPP_ASSERT(outBlocks);
1632  CRYPTOPP_ASSERT(length >= 8);
1633 
1634  const size_t blockSize = 8;
1635  const size_t xmmBlockSize = 16;
1636 
1637  size_t inIncrement = (flags & (BT_InBlockIsCounter | BT_DontIncrementInOutPointers)) ? 0 : xmmBlockSize;
1638  size_t xorIncrement = (xorBlocks != NULLPTR) ? xmmBlockSize : 0;
1639  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : xmmBlockSize;
1640 
1641  // Clang and Coverity are generating findings using xorBlocks as a flag.
1642  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
1643  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
1644 
1645  if (flags & BT_ReverseDirection)
1646  {
1647  inBlocks = PtrAdd(inBlocks, length - xmmBlockSize);
1648  xorBlocks = PtrAdd(xorBlocks, length - xmmBlockSize);
1649  outBlocks = PtrAdd(outBlocks, length - xmmBlockSize);
1650  inIncrement = 0 - inIncrement;
1651  xorIncrement = 0 - xorIncrement;
1652  outIncrement = 0 - outIncrement;
1653  }
1654 
1655  if (flags & BT_AllowParallel)
1656  {
1657  while (length >= 4*xmmBlockSize)
1658  {
1659  __m128i block0, block1, block2, block3;
1660  if (flags & BT_InBlockIsCounter)
1661  {
1662  // Increment of 1 and 2 in big-endian compatible with the ctr byte array.
1663  const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
1664  const __m128i s_two = _mm_set_epi32(2<<24, 0, 2<<24, 0);
1665  double temp[2];
1666 
1667  // For 64-bit block ciphers we need to load the CTR block, which is 8 bytes.
1668  // After the dup load we have two counters in the XMM word. Then we need
1669  // to increment the low ctr by 0 and the high ctr by 1.
1670  std::memcpy(temp, inBlocks, blockSize);
1671  block0 = _mm_add_epi32(s_one, _mm_castpd_si128(_mm_loaddup_pd(temp)));
1672 
1673  // After initial increment of {0,1} remaining counters increment by {2,2}.
1674  block1 = _mm_add_epi32(s_two, block0);
1675  block2 = _mm_add_epi32(s_two, block1);
1676  block3 = _mm_add_epi32(s_two, block2);
1677 
1678  // Store the next counter. When BT_InBlockIsCounter is set then
1679  // inBlocks is backed by m_counterArray which is non-const.
1680  _mm_store_sd(temp, _mm_castsi128_pd(_mm_add_epi64(s_two, block3)));
1681  std::memcpy(const_cast<byte*>(inBlocks), temp, blockSize);
1682  }
1683  else
1684  {
1685  block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1686  inBlocks = PtrAdd(inBlocks, inIncrement);
1687  block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1688  inBlocks = PtrAdd(inBlocks, inIncrement);
1689  block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1690  inBlocks = PtrAdd(inBlocks, inIncrement);
1691  block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
1692  inBlocks = PtrAdd(inBlocks, inIncrement);
1693  }
1694 
1695  if (xorInput)
1696  {
1697  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1698  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1699  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1700  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1701  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1702  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1703  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1704  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1705  }
1706 
1707  func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
1708 
1709  if (xorOutput)
1710  {
1711  block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1712  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1713  block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1714  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1715  block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1716  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1717  block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
1718  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1719  }
1720 
1721  _mm_storeu_si128(M128_CAST(outBlocks), block0);
1722  outBlocks = PtrAdd(outBlocks, outIncrement);
1723  _mm_storeu_si128(M128_CAST(outBlocks), block1);
1724  outBlocks = PtrAdd(outBlocks, outIncrement);
1725  _mm_storeu_si128(M128_CAST(outBlocks), block2);
1726  outBlocks = PtrAdd(outBlocks, outIncrement);
1727  _mm_storeu_si128(M128_CAST(outBlocks), block3);
1728  outBlocks = PtrAdd(outBlocks, outIncrement);
1729 
1730  length -= 4*xmmBlockSize;
1731  }
1732  }
1733 
1734  if (length)
1735  {
1736  // Adjust to real block size
1737  if (flags & BT_ReverseDirection)
1738  {
1739  inIncrement += inIncrement ? blockSize : 0;
1740  xorIncrement += xorIncrement ? blockSize : 0;
1741  outIncrement += outIncrement ? blockSize : 0;
1742  inBlocks = PtrSub(inBlocks, inIncrement);
1743  xorBlocks = PtrSub(xorBlocks, xorIncrement);
1744  outBlocks = PtrSub(outBlocks, outIncrement);
1745  }
1746  else
1747  {
1748  inIncrement -= inIncrement ? blockSize : 0;
1749  xorIncrement -= xorIncrement ? blockSize : 0;
1750  outIncrement -= outIncrement ? blockSize : 0;
1751  }
1752 
1753  while (length >= blockSize)
1754  {
1755  double temp[2];
1756  std::memcpy(temp, inBlocks, blockSize);
1757  __m128i block = _mm_castpd_si128(_mm_load_sd(temp));
1758 
1759  if (xorInput)
1760  {
1761  std::memcpy(temp, xorBlocks, blockSize);
1762  block = _mm_xor_si128(block, _mm_castpd_si128(_mm_load_sd(temp)));
1763  }
1764 
1765  if (flags & BT_InBlockIsCounter)
1766  const_cast<byte *>(inBlocks)[7]++;
1767 
1768  func1(block, subKeys, static_cast<unsigned int>(rounds));
1769 
1770  if (xorOutput)
1771  {
1772  std::memcpy(temp, xorBlocks, blockSize);
1773  block = _mm_xor_si128(block, _mm_castpd_si128(_mm_load_sd(temp)));
1774  }
1775 
1776  _mm_store_sd(temp, _mm_castsi128_pd(block));
1777  std::memcpy(outBlocks, temp, blockSize);
1778 
1779  inBlocks = PtrAdd(inBlocks, inIncrement);
1780  outBlocks = PtrAdd(outBlocks, outIncrement);
1781  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1782  length -= blockSize;
1783  }
1784  }
1785 
1786  return length;
1787 }
1788 
1789 NAMESPACE_END // CryptoPP
1790 
1791 #endif // CRYPTOPP_SSSE3_AVAILABLE
1792 
1793 // *********************** Altivec/Power 4 ********************** //
1794 
1795 #if defined(__ALTIVEC__)
1796 
1797 NAMESPACE_BEGIN(CryptoPP)
1798 
1799 /// \brief AdvancedProcessBlocks for 2 and 6 blocks
1800 /// \tparam F2 function to process 2 128-bit blocks
1801 /// \tparam F6 function to process 6 128-bit blocks
1802 /// \tparam W word type of the subkey table
1803 /// \details AdvancedProcessBlocks64_6x2_Altivec processes 6 and 2 Altivec SIMD words
1804 /// at a time. For a single block the template uses F2 with a zero block.
1805 /// \details The subkey type is usually word32 or word64. F2 and F6 must use the
1806 /// same word type.
1807 template <typename F2, typename F6, typename W>
1808 CRYPTOPP_INLINE size_t AdvancedProcessBlocks64_6x2_ALTIVEC(F2 func2, F6 func6,
1809  const W *subKeys, size_t rounds, const byte *inBlocks,
1810  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
1811 {
1812  CRYPTOPP_ASSERT(subKeys);
1813  CRYPTOPP_ASSERT(inBlocks);
1814  CRYPTOPP_ASSERT(outBlocks);
1815  CRYPTOPP_ASSERT(length >= 8);
1816 
1817 #if (CRYPTOPP_LITTLE_ENDIAN)
1818  enum {LowOffset=8, HighOffset=0};
1819  const uint32x4_p s_one = {1,0,0,0};
1820  const uint32x4_p s_two = {2,0,2,0};
1821 #else
1822  enum {LowOffset=8, HighOffset=0};
1823  const uint32x4_p s_one = {0,0,0,1};
1824  const uint32x4_p s_two = {0,2,0,2};
1825 #endif
1826 
1827  const size_t blockSize = 8;
1828  const size_t vsxBlockSize = 16;
1829  CRYPTOPP_ALIGN_DATA(16) uint8_t temp[16];
1830 
1831  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : vsxBlockSize;
1832  size_t xorIncrement = (xorBlocks != NULLPTR) ? vsxBlockSize : 0;
1833  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : vsxBlockSize;
1834 
1835  // Clang and Coverity are generating findings using xorBlocks as a flag.
1836  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
1837  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
1838 
1839  if (flags & BT_ReverseDirection)
1840  {
1841  inBlocks = PtrAdd(inBlocks, length - vsxBlockSize);
1842  xorBlocks = PtrAdd(xorBlocks, length - vsxBlockSize);
1843  outBlocks = PtrAdd(outBlocks, length - vsxBlockSize);
1844  inIncrement = 0-inIncrement;
1845  xorIncrement = 0-xorIncrement;
1846  outIncrement = 0-outIncrement;
1847  }
1848 
1849  if (flags & BT_AllowParallel)
1850  {
1851  while (length >= 6*vsxBlockSize)
1852  {
1853  uint32x4_p block0, block1, block2, block3, block4, block5;
1854  if (flags & BT_InBlockIsCounter)
1855  {
1856  // There is no easy way to load 8-bytes into a vector. It is
1857  // even harder without POWER8 due to lack of 64-bit elements.
1858  std::memcpy(temp+LowOffset, inBlocks, 8);
1859  std::memcpy(temp+HighOffset, inBlocks, 8);
1860  uint32x4_p ctr = (uint32x4_p)VecLoadBE(temp);
1861 
1862  // For 64-bit block ciphers we need to load the CTR block,
1863  // which is 8 bytes. After the dup load we have two counters
1864  // in the Altivec word. Then we need to increment the low ctr
1865  // by 0 and the high ctr by 1.
1866  block0 = VecAdd(s_one, ctr);
1867 
1868  // After initial increment of {0,1} remaining counters
1869  // increment by {2,2}.
1870  block1 = VecAdd(s_two, block0);
1871  block2 = VecAdd(s_two, block1);
1872  block3 = VecAdd(s_two, block2);
1873  block4 = VecAdd(s_two, block3);
1874  block5 = VecAdd(s_two, block4);
1875 
1876  // Update the counter in the caller.
1877  const_cast<byte*>(inBlocks)[7] += 12;
1878  }
1879  else
1880  {
1881  block0 = VecLoadBE(inBlocks);
1882  inBlocks = PtrAdd(inBlocks, inIncrement);
1883  block1 = VecLoadBE(inBlocks);
1884  inBlocks = PtrAdd(inBlocks, inIncrement);
1885  block2 = VecLoadBE(inBlocks);
1886  inBlocks = PtrAdd(inBlocks, inIncrement);
1887  block3 = VecLoadBE(inBlocks);
1888  inBlocks = PtrAdd(inBlocks, inIncrement);
1889  block4 = VecLoadBE(inBlocks);
1890  inBlocks = PtrAdd(inBlocks, inIncrement);
1891  block5 = VecLoadBE(inBlocks);
1892  inBlocks = PtrAdd(inBlocks, inIncrement);
1893  }
1894 
1895  if (xorInput)
1896  {
1897  block0 = VecXor(block0, VecLoadBE(xorBlocks));
1898  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1899  block1 = VecXor(block1, VecLoadBE(xorBlocks));
1900  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1901  block2 = VecXor(block2, VecLoadBE(xorBlocks));
1902  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1903  block3 = VecXor(block3, VecLoadBE(xorBlocks));
1904  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1905  block4 = VecXor(block4, VecLoadBE(xorBlocks));
1906  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1907  block5 = VecXor(block5, VecLoadBE(xorBlocks));
1908  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1909  }
1910 
1911  func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
1912 
1913  if (xorOutput)
1914  {
1915  block0 = VecXor(block0, VecLoadBE(xorBlocks));
1916  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1917  block1 = VecXor(block1, VecLoadBE(xorBlocks));
1918  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1919  block2 = VecXor(block2, VecLoadBE(xorBlocks));
1920  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1921  block3 = VecXor(block3, VecLoadBE(xorBlocks));
1922  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1923  block4 = VecXor(block4, VecLoadBE(xorBlocks));
1924  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1925  block5 = VecXor(block5, VecLoadBE(xorBlocks));
1926  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1927  }
1928 
1929  VecStoreBE(block0, outBlocks);
1930  outBlocks = PtrAdd(outBlocks, outIncrement);
1931  VecStoreBE(block1, outBlocks);
1932  outBlocks = PtrAdd(outBlocks, outIncrement);
1933  VecStoreBE(block2, outBlocks);
1934  outBlocks = PtrAdd(outBlocks, outIncrement);
1935  VecStoreBE(block3, outBlocks);
1936  outBlocks = PtrAdd(outBlocks, outIncrement);
1937  VecStoreBE(block4, outBlocks);
1938  outBlocks = PtrAdd(outBlocks, outIncrement);
1939  VecStoreBE(block5, outBlocks);
1940  outBlocks = PtrAdd(outBlocks, outIncrement);
1941 
1942  length -= 6*vsxBlockSize;
1943  }
1944 
1945  while (length >= 2*vsxBlockSize)
1946  {
1947  uint32x4_p block0, block1;
1948  if (flags & BT_InBlockIsCounter)
1949  {
1950  // There is no easy way to load 8-bytes into a vector. It is
1951  // even harder without POWER8 due to lack of 64-bit elements.
1952  std::memcpy(temp+LowOffset, inBlocks, 8);
1953  std::memcpy(temp+HighOffset, inBlocks, 8);
1954  uint32x4_p ctr = (uint32x4_p)VecLoadBE(temp);
1955 
1956  // For 64-bit block ciphers we need to load the CTR block,
1957  // which is 8 bytes. After the dup load we have two counters
1958  // in the Altivec word. Then we need to increment the low ctr
1959  // by 0 and the high ctr by 1.
1960  block0 = VecAdd(s_one, ctr);
1961 
1962  // After initial increment of {0,1} remaining counters
1963  // increment by {2,2}.
1964  block1 = VecAdd(s_two, block0);
1965 
1966  // Update the counter in the caller.
1967  const_cast<byte*>(inBlocks)[7] += 4;
1968  }
1969  else
1970  {
1971  block0 = VecLoadBE(inBlocks);
1972  inBlocks = PtrAdd(inBlocks, inIncrement);
1973  block1 = VecLoadBE(inBlocks);
1974  inBlocks = PtrAdd(inBlocks, inIncrement);
1975  }
1976 
1977  if (xorInput)
1978  {
1979  block0 = VecXor(block0, VecLoadBE(xorBlocks));
1980  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1981  block1 = VecXor(block1, VecLoadBE(xorBlocks));
1982  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1983  }
1984 
1985  func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
1986 
1987  if (xorOutput)
1988  {
1989  block0 = VecXor(block0, VecLoadBE(xorBlocks));
1990  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1991  block1 = VecXor(block1, VecLoadBE(xorBlocks));
1992  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
1993  }
1994 
1995  VecStoreBE(block0, outBlocks);
1996  outBlocks = PtrAdd(outBlocks, outIncrement);
1997  VecStoreBE(block1, outBlocks);
1998  outBlocks = PtrAdd(outBlocks, outIncrement);
1999 
2000  length -= 2*vsxBlockSize;
2001  }
2002  }
2003 
2004  if (length)
2005  {
2006  // Adjust to real block size
2007  if (flags & BT_ReverseDirection)
2008  {
2009  inIncrement += inIncrement ? blockSize : 0;
2010  xorIncrement += xorIncrement ? blockSize : 0;
2011  outIncrement += outIncrement ? blockSize : 0;
2012  inBlocks = PtrSub(inBlocks, inIncrement);
2013  xorBlocks = PtrSub(xorBlocks, xorIncrement);
2014  outBlocks = PtrSub(outBlocks, outIncrement);
2015  }
2016  else
2017  {
2018  inIncrement -= inIncrement ? blockSize : 0;
2019  xorIncrement -= xorIncrement ? blockSize : 0;
2020  outIncrement -= outIncrement ? blockSize : 0;
2021  }
2022 
2023  while (length >= blockSize)
2024  {
2025  uint32x4_p block, zero = {0};
2026 
2027  // There is no easy way to load 8-bytes into a vector. It is
2028  // even harder without POWER8 due to lack of 64-bit elements.
2029  // The high 8 bytes are "don't care" but it if we don't
2030  // initialize the block then it generates warnings.
2031  std::memcpy(temp+LowOffset, inBlocks, 8);
2032  std::memcpy(temp+HighOffset, inBlocks, 8); // don't care
2033  block = (uint32x4_p)VecLoadBE(temp);
2034 
2035  if (xorInput)
2036  {
2037  std::memcpy(temp+LowOffset, xorBlocks, 8);
2038  std::memcpy(temp+HighOffset, xorBlocks, 8); // don't care
2039  uint32x4_p x = (uint32x4_p)VecLoadBE(temp);
2040  block = VecXor(block, x);
2041  }
2042 
2043  // Update the counter in the caller.
2044  if (flags & BT_InBlockIsCounter)
2045  const_cast<byte *>(inBlocks)[7]++;
2046 
2047  func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
2048 
2049  if (xorOutput)
2050  {
2051  std::memcpy(temp+LowOffset, xorBlocks, 8);
2052  std::memcpy(temp+HighOffset, xorBlocks, 8); // don't care
2053  uint32x4_p x = (uint32x4_p)VecLoadBE(temp);
2054  block = VecXor(block, x);
2055  }
2056 
2057  VecStoreBE(block, temp);
2058  std::memcpy(outBlocks, temp+LowOffset, 8);
2059 
2060  inBlocks = PtrAdd(inBlocks, inIncrement);
2061  outBlocks = PtrAdd(outBlocks, outIncrement);
2062  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2063  length -= blockSize;
2064  }
2065  }
2066 
2067  return length;
2068 }
2069 
2070 /// \brief AdvancedProcessBlocks for 1 and 4 blocks
2071 /// \tparam F1 function to process 1 128-bit block
2072 /// \tparam F4 function to process 4 128-bit blocks
2073 /// \tparam W word type of the subkey table
2074 /// \details AdvancedProcessBlocks128_4x1_ALTIVEC processes 4 and 1 Altivec SIMD words
2075 /// at a time.
2076 /// \details The subkey type is usually word32 or word64. F1 and F4 must use the
2077 /// same word type.
2078 template <typename F1, typename F4, typename W>
2079 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_4x1_ALTIVEC(F1 func1, F4 func4,
2080  const W *subKeys, size_t rounds, const byte *inBlocks,
2081  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
2082 {
2083  CRYPTOPP_ASSERT(subKeys);
2084  CRYPTOPP_ASSERT(inBlocks);
2085  CRYPTOPP_ASSERT(outBlocks);
2086  CRYPTOPP_ASSERT(length >= 16);
2087 
2088 #if (CRYPTOPP_LITTLE_ENDIAN)
2089  const uint32x4_p s_one = {1,0,0,0};
2090 #else
2091  const uint32x4_p s_one = {0,0,0,1};
2092 #endif
2093 
2094  const size_t blockSize = 16;
2095  // const size_t vsxBlockSize = 16;
2096 
2097  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
2098  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
2099  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
2100 
2101  // Clang and Coverity are generating findings using xorBlocks as a flag.
2102  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
2103  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
2104 
2105  if (flags & BT_ReverseDirection)
2106  {
2107  inBlocks = PtrAdd(inBlocks, length - blockSize);
2108  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
2109  outBlocks = PtrAdd(outBlocks, length - blockSize);
2110  inIncrement = 0-inIncrement;
2111  xorIncrement = 0-xorIncrement;
2112  outIncrement = 0-outIncrement;
2113  }
2114 
2115  if (flags & BT_AllowParallel)
2116  {
2117  while (length >= 4*blockSize)
2118  {
2119  uint32x4_p block0, block1, block2, block3;
2120 
2121  if (flags & BT_InBlockIsCounter)
2122  {
2123  block0 = VecLoadBE(inBlocks);
2124  block1 = VecAdd(block0, s_one);
2125  block2 = VecAdd(block1, s_one);
2126  block3 = VecAdd(block2, s_one);
2127 
2128  // Hack due to big-endian loads used by POWER8 (and maybe ARM-BE).
2129  // CTR_ModePolicy::OperateKeystream is wired such that after
2130  // returning from this function CTR_ModePolicy will detect wrap on
2131  // on the last counter byte and increment the next to last byte.
2132  // The problem is, with a big-endian load, inBlocks[15] is really
2133  // located at index 15. The vector addition using a 32-bit element
2134  // generates a carry into inBlocks[14] and then CTR_ModePolicy
2135  // increments inBlocks[14] too.
2136  const_cast<byte*>(inBlocks)[15] += 6;
2137  }
2138  else
2139  {
2140  block0 = VecLoadBE(inBlocks);
2141  inBlocks = PtrAdd(inBlocks, inIncrement);
2142  block1 = VecLoadBE(inBlocks);
2143  inBlocks = PtrAdd(inBlocks, inIncrement);
2144  block2 = VecLoadBE(inBlocks);
2145  inBlocks = PtrAdd(inBlocks, inIncrement);
2146  block3 = VecLoadBE(inBlocks);
2147  inBlocks = PtrAdd(inBlocks, inIncrement);
2148  }
2149 
2150  if (xorInput)
2151  {
2152  block0 = VecXor(block0, VecLoadBE(xorBlocks));
2153  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2154  block1 = VecXor(block1, VecLoadBE(xorBlocks));
2155  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2156  block2 = VecXor(block2, VecLoadBE(xorBlocks));
2157  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2158  block3 = VecXor(block3, VecLoadBE(xorBlocks));
2159  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2160  }
2161 
2162  func4(block0, block1, block2, block3, subKeys, rounds);
2163 
2164  if (xorOutput)
2165  {
2166  block0 = VecXor(block0, VecLoadBE(xorBlocks));
2167  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2168  block1 = VecXor(block1, VecLoadBE(xorBlocks));
2169  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2170  block2 = VecXor(block2, VecLoadBE(xorBlocks));
2171  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2172  block3 = VecXor(block3, VecLoadBE(xorBlocks));
2173  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2174  }
2175 
2176  VecStoreBE(block0, outBlocks);
2177  outBlocks = PtrAdd(outBlocks, outIncrement);
2178  VecStoreBE(block1, outBlocks);
2179  outBlocks = PtrAdd(outBlocks, outIncrement);
2180  VecStoreBE(block2, outBlocks);
2181  outBlocks = PtrAdd(outBlocks, outIncrement);
2182  VecStoreBE(block3, outBlocks);
2183  outBlocks = PtrAdd(outBlocks, outIncrement);
2184 
2185  length -= 4*blockSize;
2186  }
2187  }
2188 
2189  while (length >= blockSize)
2190  {
2191  uint32x4_p block = VecLoadBE(inBlocks);
2192 
2193  if (xorInput)
2194  block = VecXor(block, VecLoadBE(xorBlocks));
2195 
2196  if (flags & BT_InBlockIsCounter)
2197  const_cast<byte *>(inBlocks)[15]++;
2198 
2199  func1(block, subKeys, rounds);
2200 
2201  if (xorOutput)
2202  block = VecXor(block, VecLoadBE(xorBlocks));
2203 
2204  VecStoreBE(block, outBlocks);
2205 
2206  inBlocks = PtrAdd(inBlocks, inIncrement);
2207  outBlocks = PtrAdd(outBlocks, outIncrement);
2208  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2209  length -= blockSize;
2210  }
2211 
2212  return length;
2213 }
2214 
2215 /// \brief AdvancedProcessBlocks for 1 and 6 blocks
2216 /// \tparam F1 function to process 1 128-bit block
2217 /// \tparam F6 function to process 6 128-bit blocks
2218 /// \tparam W word type of the subkey table
2219 /// \details AdvancedProcessBlocks128_6x1_ALTIVEC processes 6 and 1 Altivec SIMD words
2220 /// at a time.
2221 /// \details The subkey type is usually word32 or word64. F1 and F6 must use the
2222 /// same word type.
2223 template <typename F1, typename F6, typename W>
2224 CRYPTOPP_INLINE size_t AdvancedProcessBlocks128_6x1_ALTIVEC(F1 func1, F6 func6,
2225  const W *subKeys, size_t rounds, const byte *inBlocks,
2226  const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
2227 {
2228  CRYPTOPP_ASSERT(subKeys);
2229  CRYPTOPP_ASSERT(inBlocks);
2230  CRYPTOPP_ASSERT(outBlocks);
2231  CRYPTOPP_ASSERT(length >= 16);
2232 
2233 #if (CRYPTOPP_LITTLE_ENDIAN)
2234  const uint32x4_p s_one = {1,0,0,0};
2235 #else
2236  const uint32x4_p s_one = {0,0,0,1};
2237 #endif
2238 
2239  const size_t blockSize = 16;
2240  // const size_t vsxBlockSize = 16;
2241 
2242  size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
2243  size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
2244  size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
2245 
2246  // Clang and Coverity are generating findings using xorBlocks as a flag.
2247  const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
2248  const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
2249 
2250  if (flags & BT_ReverseDirection)
2251  {
2252  inBlocks = PtrAdd(inBlocks, length - blockSize);
2253  xorBlocks = PtrAdd(xorBlocks, length - blockSize);
2254  outBlocks = PtrAdd(outBlocks, length - blockSize);
2255  inIncrement = 0-inIncrement;
2256  xorIncrement = 0-xorIncrement;
2257  outIncrement = 0-outIncrement;
2258  }
2259 
2260  if (flags & BT_AllowParallel)
2261  {
2262  while (length >= 6*blockSize)
2263  {
2264  uint32x4_p block0, block1, block2, block3, block4, block5;
2265 
2266  if (flags & BT_InBlockIsCounter)
2267  {
2268  block0 = VecLoadBE(inBlocks);
2269  block1 = VecAdd(block0, s_one);
2270  block2 = VecAdd(block1, s_one);
2271  block3 = VecAdd(block2, s_one);
2272  block4 = VecAdd(block3, s_one);
2273  block5 = VecAdd(block4, s_one);
2274 
2275  // Hack due to big-endian loads used by POWER8 (and maybe ARM-BE).
2276  // CTR_ModePolicy::OperateKeystream is wired such that after
2277  // returning from this function CTR_ModePolicy will detect wrap on
2278  // on the last counter byte and increment the next to last byte.
2279  // The problem is, with a big-endian load, inBlocks[15] is really
2280  // located at index 15. The vector addition using a 32-bit element
2281  // generates a carry into inBlocks[14] and then CTR_ModePolicy
2282  // increments inBlocks[14] too.
2283  //
2284  // To find this bug we needed a test case with a ctr of 0xNN...FA.
2285  // The last octet is 0xFA and adding 6 creates the wrap to trigger
2286  // the issue. If the last octet was 0xFC then 4 would trigger it.
2287  // We dumb-lucked into the test with SPECK-128. The test case of
2288  // interest is the one with IV 348ECA9766C09F04 826520DE47A212FA.
2289  uint8x16_p temp = VecAdd((uint8x16_p)block5, (uint8x16_p)s_one);
2290  VecStoreBE(temp, const_cast<byte*>(inBlocks));
2291  }
2292  else
2293  {
2294  block0 = VecLoadBE(inBlocks);
2295  inBlocks = PtrAdd(inBlocks, inIncrement);
2296  block1 = VecLoadBE(inBlocks);
2297  inBlocks = PtrAdd(inBlocks, inIncrement);
2298  block2 = VecLoadBE(inBlocks);
2299  inBlocks = PtrAdd(inBlocks, inIncrement);
2300  block3 = VecLoadBE(inBlocks);
2301  inBlocks = PtrAdd(inBlocks, inIncrement);
2302  block4 = VecLoadBE(inBlocks);
2303  inBlocks = PtrAdd(inBlocks, inIncrement);
2304  block5 = VecLoadBE(inBlocks);
2305  inBlocks = PtrAdd(inBlocks, inIncrement);
2306  }
2307 
2308  if (xorInput)
2309  {
2310  block0 = VecXor(block0, VecLoadBE(xorBlocks));
2311  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2312  block1 = VecXor(block1, VecLoadBE(xorBlocks));
2313  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2314  block2 = VecXor(block2, VecLoadBE(xorBlocks));
2315  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2316  block3 = VecXor(block3, VecLoadBE(xorBlocks));
2317  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2318  block4 = VecXor(block4, VecLoadBE(xorBlocks));
2319  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2320  block5 = VecXor(block5, VecLoadBE(xorBlocks));
2321  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2322  }
2323 
2324  func6(block0, block1, block2, block3, block4, block5, subKeys, rounds);
2325 
2326  if (xorOutput)
2327  {
2328  block0 = VecXor(block0, VecLoadBE(xorBlocks));
2329  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2330  block1 = VecXor(block1, VecLoadBE(xorBlocks));
2331  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2332  block2 = VecXor(block2, VecLoadBE(xorBlocks));
2333  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2334  block3 = VecXor(block3, VecLoadBE(xorBlocks));
2335  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2336  block4 = VecXor(block4, VecLoadBE(xorBlocks));
2337  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2338  block5 = VecXor(block5, VecLoadBE(xorBlocks));
2339  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2340  }
2341 
2342  VecStoreBE(block0, outBlocks);
2343  outBlocks = PtrAdd(outBlocks, outIncrement);
2344  VecStoreBE(block1, outBlocks);
2345  outBlocks = PtrAdd(outBlocks, outIncrement);
2346  VecStoreBE(block2, outBlocks);
2347  outBlocks = PtrAdd(outBlocks, outIncrement);
2348  VecStoreBE(block3, outBlocks);
2349  outBlocks = PtrAdd(outBlocks, outIncrement);
2350  VecStoreBE(block4, outBlocks);
2351  outBlocks = PtrAdd(outBlocks, outIncrement);
2352  VecStoreBE(block5, outBlocks);
2353  outBlocks = PtrAdd(outBlocks, outIncrement);
2354 
2355  length -= 6*blockSize;
2356  }
2357  }
2358 
2359  while (length >= blockSize)
2360  {
2361  uint32x4_p block = VecLoadBE(inBlocks);
2362 
2363  if (xorInput)
2364  block = VecXor(block, VecLoadBE(xorBlocks));
2365 
2366  if (flags & BT_InBlockIsCounter)
2367  const_cast<byte *>(inBlocks)[15]++;
2368 
2369  func1(block, subKeys, rounds);
2370 
2371  if (xorOutput)
2372  block = VecXor(block, VecLoadBE(xorBlocks));
2373 
2374  VecStoreBE(block, outBlocks);
2375 
2376  inBlocks = PtrAdd(inBlocks, inIncrement);
2377  outBlocks = PtrAdd(outBlocks, outIncrement);
2378  xorBlocks = PtrAdd(xorBlocks, xorIncrement);
2379  length -= blockSize;
2380  }
2381 
2382  return length;
2383 }
2384 
2385 NAMESPACE_END // CryptoPP
2386 
2387 #endif // __ALTIVEC__
2388 
2389 #endif // CRYPTOPP_ADVANCED_SIMD_TEMPLATES
BlockTransformation::BT_AllowParallel
Allow parallel transformations.
Definition: cryptlib.h:897
misc.h
Utility functions for the Crypto++ library.
config.h
Library configuration file.
BlockTransformation::BT_DontIncrementInOutPointers
should not modify block pointers
Definition: cryptlib.h:891
stdcpp.h
Common C++ header files.
VecAdd
T1 VecAdd(const T1 vec1, const T2 vec2)
Add two vectors.
Definition: ppc_simd.h:963
uint32x4_p
__vector unsigned int uint32x4_p
Vector of 32-bit elements.
Definition: ppc_simd.h:128
ppc_simd.h
Support functions for PowerPC and vector operations.
VecStoreBE
void VecStoreBE(const T data, byte dest[16])
Stores a vector to a byte array.
Definition: ppc_simd.h:746
CRYPTOPP_ASSERT
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:60
VecXor
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
Definition: ppc_simd.h:945
PtrSub
PTR PtrSub(PTR pointer, OFF offset)
Create a pointer with an offset.
Definition: misc.h:384
PtrAdd
PTR PtrAdd(PTR pointer, OFF offset)
Create a pointer with an offset.
Definition: misc.h:371
BlockTransformation::BT_XorInput
Xor inputs before transformation.
Definition: cryptlib.h:893
VecLoadBE
uint32x4_p VecLoadBE(const byte src[16])
Loads a vector from a byte array.
Definition: ppc_simd.h:438
BlockTransformation::BT_ReverseDirection
perform the transformation in reverse
Definition: cryptlib.h:895
CryptoPP
Crypto++ library namespace.
uint8x16_p
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
Definition: ppc_simd.h:118
Generated on Tue Dec 25 2018 16:18:37 for Crypto++ by   doxygen 1.8.14