17 #if defined(CRYPTOPP_DEBUG)
19 # include "threefish.h"
23 #if defined(__SSE2__) || defined(_M_X64)
24 # include <emmintrin.h>
27 #if defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)
28 # if (CRYPTOPP_ARM_NEON_HEADER)
29 # include <arm_neon.h>
33 #if defined(__ALTIVEC__)
37 ANONYMOUS_NAMESPACE_BEGIN
41 #if defined(CRYPTOPP_DEBUG) && !defined(CRYPTOPP_DOXYGEN_PROCESSING)
44 using CryptoPP::XTS_Mode;
45 using CryptoPP::Threefish512;
47 void Modes_TestInstantiations()
49 XTS_Mode<AES>::Encryption m0;
50 XTS_Mode<AES>::Decryption m1;
51 XTS_Mode<AES>::Encryption m2;
52 XTS_Mode<AES>::Decryption m3;
54 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
55 XTS_Mode<Threefish512>::Encryption m4;
56 XTS_Mode<Threefish512>::Decryption m5;
59 #endif // CRYPTOPP_DEBUG
61 inline void XorBuffer(
byte *output,
const byte *input,
const byte *mask,
size_t count)
65 #if defined(CRYPTOPP_DISABLE_ASM)
66 xorbuf(output, input, mask, count);
68 #elif defined(__SSE2__) || defined(_M_X64)
69 for (
size_t i=0; i<count; i+=16)
75 #elif defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)
76 for (
size_t i=0; i<count; i+=16)
77 vst1q_u8(output+i, veorq_u8(vld1q_u8(input+i), vld1q_u8(mask+i)));
79 #elif defined(__ALTIVEC__)
80 for (
size_t i=0; i<count; i+=16)
84 xorbuf(output, input, mask, count);
88 inline void XorBuffer(
byte *buf,
const byte *mask,
size_t count)
90 XorBuffer(buf, buf, mask, count);
94 inline void GF_Double(
byte *out,
const byte* in,
unsigned int len)
96 #if defined(_M_X64) || defined(_M_ARM64) || defined(_LP64) || defined(__LP64__)
98 for (
size_t i=0, idx=0; i<len/8; ++i, idx+=8)
101 word64 y = (x >> 63); x = (x << 1) + carry;
107 for (
size_t i=0, idx=0; i<len/4; ++i, idx+=4)
110 word32 y = (x >> 31); x = (x << 1) + carry;
116 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
129 const size_t LEIDX = 16-1;
137 const size_t LEIDX = 32-1;
146 const size_t LEIDX = 64-1;
155 const size_t LEIDX = 128-1;
157 k[LEIDX-126] ^= 0x00;
158 k[LEIDX-127] ^= 0x43;
174 #endif // CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
177 inline void GF_Double(
byte *inout,
unsigned int len)
179 GF_Double(inout, inout, len);
182 ANONYMOUS_NAMESPACE_END
188 #if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS
190 if (length < 16 || length > 128 || !
IsPowerOf2(length))
191 throw InvalidArgument(AlgorithmName() +
": block size of underlying block cipher is not valid");
195 throw InvalidArgument(AlgorithmName() +
": block size of underlying block cipher is not 16");
202 if (!GetBlockCipher().IsValidKeyLength((length+1)/2))
208 ThrowIfInvalidKeyLength(length);
211 const size_t klen = length/2;
212 AccessBlockCipher().SetKey(key+0, klen, params);
213 AccessTweakCipher().SetKey(key+klen, klen, params);
218 const byte *iv = GetIVAndThrowIfInvalid(params, ivLength);
219 Resynchronize(iv, (
int)ivLength);
225 std::memcpy(m_xregister, m_register, ivLength);
226 GetTweakCipher().ProcessBlock(m_xregister);
232 PutWord<word64>(
false, order, iv, sector);
233 std::memset(iv+8, 0x00, iv.
size()-8);
236 std::memcpy(m_xregister, iv, iv.
size());
237 GetTweakCipher().ProcessBlock(m_xregister);
240 void XTS_ModeBase::ResizeBuffers()
242 BlockOrientedCipherModeBase::ResizeBuffers();
243 m_xworkspace.New(GetBlockCipher().
BlockSize()*ParallelBlocks);
244 m_xregister.New(GetBlockCipher().
BlockSize()*ParallelBlocks);
257 enum { lastParallelBlock = ParallelBlocks-1 };
258 const unsigned int blockSize = GetBlockCipher().BlockSize();
259 const size_t parallelSize = blockSize*ParallelBlocks;
262 while (length >= parallelSize)
265 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
266 GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);
267 GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);
269 if (ParallelBlocks > 4)
271 GF_Double(m_xregister+4*blockSize, m_xregister+3*blockSize, blockSize);
272 GF_Double(m_xregister+5*blockSize, m_xregister+4*blockSize, blockSize);
273 GF_Double(m_xregister+6*blockSize, m_xregister+5*blockSize, blockSize);
274 GF_Double(m_xregister+7*blockSize, m_xregister+6*blockSize, blockSize);
276 if (ParallelBlocks > 8)
278 GF_Double(m_xregister+8*blockSize, m_xregister+7*blockSize, blockSize);
279 GF_Double(m_xregister+9*blockSize, m_xregister+8*blockSize, blockSize);
280 GF_Double(m_xregister+10*blockSize, m_xregister+9*blockSize, blockSize);
281 GF_Double(m_xregister+11*blockSize, m_xregister+10*blockSize, blockSize);
285 XorBuffer(m_xworkspace, inString, m_xregister, parallelSize);
288 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
292 GF_Double(m_xregister+0, m_xregister+lastParallelBlock*blockSize, blockSize);
294 inString += parallelSize;
295 outString += parallelSize;
296 length -= parallelSize;
300 while (ParallelBlocks == 12 && length >= blockSize*4)
303 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
304 GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);
305 GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);
308 XorBuffer(m_xworkspace, inString, m_xregister, blockSize*4);
311 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
315 GF_Double(m_xregister+0, m_xregister+3*blockSize, blockSize);
317 inString += blockSize*4;
318 outString += blockSize*4;
319 length -= blockSize*4;
323 while (ParallelBlocks == 8 && length >= blockSize*2)
326 GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);
329 XorBuffer(m_xworkspace, inString, m_xregister, blockSize*2);
332 GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,
336 GF_Double(m_xregister+0, m_xregister+1*blockSize, blockSize);
338 inString += blockSize*2;
339 outString += blockSize*2;
340 length -= blockSize*2;
347 XorBuffer(m_xworkspace, inString, m_xregister, blockSize);
350 GetBlockCipher().ProcessBlock(m_xworkspace);
353 XorBuffer(outString, m_xworkspace, m_xregister, blockSize);
356 GF_Double(m_xregister, blockSize);
358 inString += blockSize;
359 outString += blockSize;
370 throw InvalidArgument(
"XTS: message is too short for ciphertext stealing");
372 if (IsForwardTransformation())
373 return ProcessLastPlainBlock(outString, outLength, inString, inLength);
375 return ProcessLastCipherBlock(outString, outLength, inString, inLength);
378 size_t XTS_ModeBase::ProcessLastPlainBlock(
byte *outString,
size_t outLength,
const byte *inString,
size_t inLength)
383 const unsigned int blockSize = GetBlockCipher().BlockSize();
384 const size_t blocks = inLength / blockSize;
385 const size_t tail = inLength % blockSize;
386 outLength = inLength;
391 ProcessData(outString, inString, inLength);
397 const size_t head = (blocks-1)*blockSize;
398 ProcessData(outString, inString, inLength-head);
401 inString += head; inLength -= head;
407 XorBuffer(m_xworkspace, inString, m_xregister, blockSize);
410 GetBlockCipher().ProcessBlock(m_xworkspace);
413 XorBuffer(outString, m_xworkspace, m_xregister, blockSize);
416 GF_Double(m_xregister, blockSize);
420 inString += blockSize;
421 outString += blockSize;
422 const size_t len = inLength-blockSize;
425 std::memcpy(m_xworkspace, inString, len);
427 std::memcpy(outString, outString-blockSize, len);
429 std::memcpy(m_xworkspace+len, outString-blockSize+len, blockSize-len);
432 XorBuffer(m_xworkspace, m_xregister, blockSize);
435 GetBlockCipher().ProcessBlock(m_xworkspace);
438 XorBuffer(outString-blockSize, m_xworkspace, m_xregister, blockSize);
443 size_t XTS_ModeBase::ProcessLastCipherBlock(
byte *outString,
size_t outLength,
const byte *inString,
size_t inLength)
448 const unsigned int blockSize = GetBlockCipher().BlockSize();
449 const size_t blocks = inLength / blockSize;
450 const size_t tail = inLength % blockSize;
451 outLength = inLength;
456 ProcessData(outString, inString, inLength);
462 const size_t head = (blocks-1)*blockSize;
463 ProcessData(outString, inString, inLength-head);
466 inString += head; inLength -= head;
469 #define poly1 (m_xregister+0*blockSize)
470 #define poly2 (m_xregister+1*blockSize)
471 GF_Double(poly2, poly1, blockSize);
475 inString += blockSize;
476 outString += blockSize;
477 const size_t len = inLength-blockSize;
480 XorBuffer(m_xworkspace, inString-blockSize, poly2, blockSize);
483 GetBlockCipher().ProcessBlock(m_xworkspace);
486 XorBuffer(m_xworkspace, poly2, blockSize);
489 std::memcpy(outString-blockSize, inString, len);
491 std::memcpy(outString, m_xworkspace, len);
493 std::memcpy(outString-blockSize+len, m_xworkspace+len, blockSize-len);
497 inString -= blockSize;
498 outString -= blockSize;
501 XorBuffer(m_xworkspace, outString, poly1, blockSize);
504 GetBlockCipher().ProcessBlock(m_xworkspace);
507 XorBuffer(outString, m_xworkspace, poly1, blockSize);