00001
00002
00003
00004
00005 #include "pch.h"
00006 #include "config.h"
00007
00008 #if CRYPTOPP_MSC_VERSION
00009 # pragma warning(disable: 4189)
00010 #endif
00011
00012 #ifndef CRYPTOPP_IMPORTS
00013 #ifndef CRYPTOPP_GENERATE_X64_MASM
00014
00015 #include "gcm.h"
00016 #include "cpu.h"
00017
00018 NAMESPACE_BEGIN(CryptoPP)
00019
00020 word16 GCM_Base::s_reductionTable[256];
00021 volatile bool GCM_Base::s_reductionTableInitialized = false;
00022
00023 void GCM_Base::GCTR::IncrementCounterBy256()
00024 {
00025 IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
00026 }
00027
00028 #if 0
00029
00030 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
00031 {
00032 word64 Z0=0, Z1=0, V0, V1;
00033
00034 typedef BlockGetAndPut<word64, BigEndian> Block;
00035 Block::Get(a)(V0)(V1);
00036
00037 for (int i=0; i<16; i++)
00038 {
00039 for (int j=0x80; j!=0; j>>=1)
00040 {
00041 int x = b[i] & j;
00042 Z0 ^= x ? V0 : 0;
00043 Z1 ^= x ? V1 : 0;
00044 x = (int)V1 & 1;
00045 V1 = (V1>>1) | (V0<<63);
00046 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00047 }
00048 }
00049 Block::Put(NULL, c)(Z0)(Z1);
00050 }
00051
00052 __m128i _mm_clmulepi64_si128(const __m128i &a, const __m128i &b, int i)
00053 {
00054 word64 A[1] = {ByteReverse(((word64*)&a)[i&1])};
00055 word64 B[1] = {ByteReverse(((word64*)&b)[i>>4])};
00056
00057 PolynomialMod2 pa((byte *)A, 8);
00058 PolynomialMod2 pb((byte *)B, 8);
00059 PolynomialMod2 c = pa*pb;
00060
00061 __m128i output;
00062 for (int i=0; i<16; i++)
00063 ((byte *)&output)[i] = c.GetByte(i);
00064 return output;
00065 }
00066 #endif
00067
00068 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00069 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
00070 {
00071 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
00072 *(__m128i *)a = _mm_xor_si128(*(__m128i *)b, *(__m128i *)c);
00073 #else
00074 asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
00075 #endif
00076 }
00077 #endif
00078
00079 inline static void Xor16(byte *a, const byte *b, const byte *c)
00080 {
00081 ((word64 *)a)[0] = ((word64 *)b)[0] ^ ((word64 *)c)[0];
00082 ((word64 *)a)[1] = ((word64 *)b)[1] ^ ((word64 *)c)[1];
00083 }
00084
00085 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00086 static CRYPTOPP_ALIGN_DATA(16) const word64 s_clmulConstants64[] = {
00087 W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
00088 W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
00089 W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
00090 static const __m128i *s_clmulConstants = (const __m128i *)s_clmulConstants64;
00091 static const unsigned int s_clmulTableSizeInBlocks = 8;
00092
00093 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2, const __m128i &r)
00094 {
00095
00096
00097
00098
00099
00100
00101
00102
00103
00104
00105
00106
00107 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301
00108 c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
00109 #else
00110 c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
00111 #endif
00112 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
00113 c0 = _mm_srli_si128(c0, 8);
00114 c0 = _mm_xor_si128(c0, c1);
00115 c0 = _mm_slli_epi64(c0, 1);
00116 c0 = _mm_clmulepi64_si128(c0, r, 0);
00117 c2 = _mm_xor_si128(c2, c0);
00118 c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
00119 c1 = _mm_unpacklo_epi64(c1, c2);
00120 c1 = _mm_srli_epi64(c1, 63);
00121 c2 = _mm_slli_epi64(c2, 1);
00122 return _mm_xor_si128(c2, c1);
00123 }
00124
00125 inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r)
00126 {
00127 __m128i c0 = _mm_clmulepi64_si128(x,h,0);
00128 __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
00129 __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
00130
00131 return CLMUL_Reduce(c0, c1, c2, r);
00132 }
00133 #endif
00134
00135 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs ¶ms)
00136 {
00137 BlockCipher &blockCipher = AccessBlockCipher();
00138 blockCipher.SetKey(userKey, keylength, params);
00139
00140 if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
00141 throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
00142
00143 int tableSize, i, j, k;
00144
00145 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00146 if (HasCLMUL())
00147 {
00148
00149 (void)params.GetIntValue(Name::TableSize(), tableSize);
00150 tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
00151 }
00152 else
00153 #endif
00154 {
00155 if (params.GetIntValue(Name::TableSize(), tableSize))
00156 tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
00157 else
00158 tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
00159
00160 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
00161
00162 tableSize = 2*1024;
00163 #endif
00164 }
00165
00166 m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
00167 byte *table = MulTable();
00168 byte *hashKey = HashKey();
00169 memset(hashKey, 0, REQUIRED_BLOCKSIZE);
00170 blockCipher.ProcessBlock(hashKey);
00171
00172 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00173 if (HasCLMUL())
00174 {
00175 const __m128i r = s_clmulConstants[0];
00176 __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)hashKey), s_clmulConstants[1]);
00177 __m128i h = h0;
00178
00179 for (i=0; i<tableSize; i+=32)
00180 {
00181 __m128i h1 = CLMUL_GF_Mul(h, h0, r);
00182 _mm_storel_epi64((__m128i *)(table+i), h);
00183 _mm_storeu_si128((__m128i *)(table+i+16), h1);
00184 _mm_storeu_si128((__m128i *)(table+i+8), h);
00185 _mm_storel_epi64((__m128i *)(table+i+8), h1);
00186 h = CLMUL_GF_Mul(h1, h0, r);
00187 }
00188
00189 return;
00190 }
00191 #endif
00192
00193 word64 V0, V1;
00194 typedef BlockGetAndPut<word64, BigEndian> Block;
00195 Block::Get(hashKey)(V0)(V1);
00196
00197 if (tableSize == 64*1024)
00198 {
00199 for (i=0; i<128; i++)
00200 {
00201 k = i%8;
00202 Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
00203
00204 int x = (int)V1 & 1;
00205 V1 = (V1>>1) | (V0<<63);
00206 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00207 }
00208
00209 for (i=0; i<16; i++)
00210 {
00211 memset(table+i*256*16, 0, 16);
00212 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00213 if (HasSSE2())
00214 for (j=2; j<=0x80; j*=2)
00215 for (k=1; k<j; k++)
00216 SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
00217 else
00218 #endif
00219 for (j=2; j<=0x80; j*=2)
00220 for (k=1; k<j; k++)
00221 Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
00222 }
00223 }
00224 else
00225 {
00226 if (!s_reductionTableInitialized)
00227 {
00228 s_reductionTable[0] = 0;
00229 word16 x = 0x01c2;
00230 s_reductionTable[1] = ByteReverse(x);
00231 for (unsigned int ii=2; ii<=0x80; ii*=2)
00232 {
00233 x <<= 1;
00234 s_reductionTable[ii] = ByteReverse(x);
00235 for (unsigned int jj=1; jj<ii; jj++)
00236 s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
00237 }
00238 s_reductionTableInitialized = true;
00239 }
00240
00241 for (i=0; i<128-24; i++)
00242 {
00243 k = i%32;
00244 if (k < 4)
00245 Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
00246 else if (k < 8)
00247 Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
00248
00249 int x = (int)V1 & 1;
00250 V1 = (V1>>1) | (V0<<63);
00251 V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
00252 }
00253
00254 for (i=0; i<4; i++)
00255 {
00256 memset(table+i*256, 0, 16);
00257 memset(table+1024+i*256, 0, 16);
00258 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00259 if (HasSSE2())
00260 for (j=2; j<=8; j*=2)
00261 for (k=1; k<j; k++)
00262 {
00263 SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
00264 SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
00265 }
00266 else
00267 #endif
00268 for (j=2; j<=8; j*=2)
00269 for (k=1; k<j; k++)
00270 {
00271 Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
00272 Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
00273 }
00274 }
00275 }
00276 }
00277
00278 inline void GCM_Base::ReverseHashBufferIfNeeded()
00279 {
00280 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00281 if (HasCLMUL())
00282 {
00283 __m128i &x = *(__m128i *)HashBuffer();
00284 x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
00285 }
00286 #endif
00287 }
00288
00289 void GCM_Base::Resync(const byte *iv, size_t len)
00290 {
00291 BlockCipher &cipher = AccessBlockCipher();
00292 byte *hashBuffer = HashBuffer();
00293
00294 if (len == 12)
00295 {
00296 memcpy(hashBuffer, iv, len);
00297 memset(hashBuffer+len, 0, 3);
00298 hashBuffer[len+3] = 1;
00299 }
00300 else
00301 {
00302 size_t origLen = len;
00303 memset(hashBuffer, 0, HASH_BLOCKSIZE);
00304
00305 if (len >= HASH_BLOCKSIZE)
00306 {
00307 len = GCM_Base::AuthenticateBlocks(iv, len);
00308 iv += (origLen - len);
00309 }
00310
00311 if (len > 0)
00312 {
00313 memcpy(m_buffer, iv, len);
00314 memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
00315 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00316 }
00317
00318 PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
00319 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00320
00321 ReverseHashBufferIfNeeded();
00322 }
00323
00324 if (m_state >= State_IVSet)
00325 m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
00326 else
00327 m_ctr.SetCipherWithIV(cipher, hashBuffer);
00328
00329 m_ctr.Seek(HASH_BLOCKSIZE);
00330
00331 memset(hashBuffer, 0, HASH_BLOCKSIZE);
00332 }
00333
00334 unsigned int GCM_Base::OptimalDataAlignment() const
00335 {
00336 return
00337 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
00338 HasSSE2() ? 16 :
00339 #endif
00340 GetBlockCipher().OptimalDataAlignment();
00341 }
00342
00343 #if CRYPTOPP_MSC_VERSION
00344 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
00345 #endif
00346
00347 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00348
00349 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
00350 extern "C" {
00351 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
00352 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
00353 }
00354 #endif
00355
00356 #ifndef CRYPTOPP_GENERATE_X64_MASM
00357
00358 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
00359 {
00360 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00361 if (HasCLMUL())
00362 {
00363 const __m128i *table = (const __m128i *)MulTable();
00364 __m128i x = _mm_load_si128((__m128i *)HashBuffer());
00365 const __m128i r = s_clmulConstants[0], bswapMask = s_clmulConstants[1], bswapMask2 = s_clmulConstants[2];
00366
00367 while (len >= 16)
00368 {
00369 size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
00370 __m128i d, d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-1)*16)), bswapMask2);;
00371 __m128i c0 = _mm_setzero_si128();
00372 __m128i c1 = _mm_setzero_si128();
00373 __m128i c2 = _mm_setzero_si128();
00374
00375 while (true)
00376 {
00377 __m128i h0 = _mm_load_si128(table+i);
00378 __m128i h1 = _mm_load_si128(table+i+1);
00379 __m128i h01 = _mm_xor_si128(h0, h1);
00380
00381 if (++i == s)
00382 {
00383 d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)data), bswapMask);
00384 d = _mm_xor_si128(d, x);
00385 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0));
00386 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
00387 d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
00388 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0));
00389 break;
00390 }
00391
00392 d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-i)*16-8)), bswapMask2);
00393 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
00394 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
00395 d2 = _mm_xor_si128(d2, d);
00396 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h01, 1));
00397
00398 if (++i == s)
00399 {
00400 d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)data), bswapMask);
00401 d = _mm_xor_si128(d, x);
00402 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
00403 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 0x11));
00404 d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
00405 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
00406 break;
00407 }
00408
00409 d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-i)*16-8)), bswapMask);
00410 c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
00411 c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
00412 d = _mm_xor_si128(d, d2);
00413 c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
00414 }
00415 data += s*16;
00416 len -= s*16;
00417
00418 c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
00419 x = CLMUL_Reduce(c0, c1, c2, r);
00420 }
00421
00422 _mm_store_si128((__m128i *)HashBuffer(), x);
00423 return len;
00424 }
00425 #endif
00426
00427 typedef BlockGetAndPut<word64, NativeByteOrder> Block;
00428 word64 *hashBuffer = (word64 *)HashBuffer();
00429
00430 switch (2*(m_buffer.size()>=64*1024)
00431 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
00432 + HasSSE2()
00433 #endif
00434 )
00435 {
00436 case 0:
00437 {
00438 byte *table = MulTable();
00439 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
00440
00441 do
00442 {
00443 word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
00444 Block::Get(data)(y0)(y1);
00445 x0 ^= y0;
00446 x1 ^= y1;
00447
00448 data += HASH_BLOCKSIZE;
00449 len -= HASH_BLOCKSIZE;
00450
00451 #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(table+(a*1024)+(b*256)+c+d*8)
00452
00453 #ifdef IS_LITTLE_ENDIAN
00454 #if CRYPTOPP_BOOL_SLOW_WORD64
00455 word32 z0 = (word32)x0;
00456 word32 z1 = (word32)(x0>>32);
00457 word32 z2 = (word32)x1;
00458 word32 z3 = (word32)(x1>>32);
00459 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
00460 #else
00461 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
00462 #endif
00463 #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
00464 #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
00465 #else
00466 #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
00467 #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
00468 #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
00469 #endif
00470
00471 #define GF_MUL_32BY128(op, a, b, c) \
00472 a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
00473 a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
00474 b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
00475 b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
00476 c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
00477 c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
00478 d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
00479 d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
00480
00481 GF_MUL_32BY128(=, 0, 0, 0)
00482 GF_MUL_32BY128(^=, 0, 1, 1)
00483 GF_MUL_32BY128(^=, 1, 0, 2)
00484 GF_MUL_32BY128(^=, 1, 1, 3)
00485
00486 word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
00487 GF_SHIFT_8(d)
00488 c0 ^= d0; c1 ^= d1;
00489 r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
00490 GF_SHIFT_8(c)
00491 b0 ^= c0; b1 ^= c1;
00492 r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
00493 GF_SHIFT_8(b)
00494 a0 ^= b0; a1 ^= b1;
00495 a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
00496 x0 = a0; x1 = a1;
00497 }
00498 while (len >= HASH_BLOCKSIZE);
00499
00500 hashBuffer[0] = x0; hashBuffer[1] = x1;
00501 return len;
00502 }
00503
00504 case 2:
00505 {
00506 byte *table = MulTable();
00507 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
00508
00509 do
00510 {
00511 word64 y0, y1, a0, a1;
00512 Block::Get(data)(y0)(y1);
00513 x0 ^= y0;
00514 x1 ^= y1;
00515
00516 data += HASH_BLOCKSIZE;
00517 len -= HASH_BLOCKSIZE;
00518
00519 #undef READ_TABLE_WORD64_COMMON
00520 #undef READ_TABLE_WORD64
00521
00522 #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(table+(a)*256*16+(c)+(d)*8)
00523
00524 #ifdef IS_LITTLE_ENDIAN
00525 #if CRYPTOPP_BOOL_SLOW_WORD64
00526 word32 z0 = (word32)x0;
00527 word32 z1 = (word32)(x0>>32);
00528 word32 z2 = (word32)x1;
00529 word32 z3 = (word32)(x1>>32);
00530 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
00531 #else
00532 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
00533 #endif
00534 #else
00535 #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
00536 #endif
00537
00538 #define GF_MUL_8BY128(op, b, c, d) \
00539 a0 op READ_TABLE_WORD64(b, c, d, 0);\
00540 a1 op READ_TABLE_WORD64(b, c, d, 1);\
00541
00542 GF_MUL_8BY128(=, 0, 0, 0)
00543 GF_MUL_8BY128(^=, 0, 0, 1)
00544 GF_MUL_8BY128(^=, 0, 0, 2)
00545 GF_MUL_8BY128(^=, 0, 0, 3)
00546 GF_MUL_8BY128(^=, 0, 1, 0)
00547 GF_MUL_8BY128(^=, 0, 1, 1)
00548 GF_MUL_8BY128(^=, 0, 1, 2)
00549 GF_MUL_8BY128(^=, 0, 1, 3)
00550 GF_MUL_8BY128(^=, 1, 2, 0)
00551 GF_MUL_8BY128(^=, 1, 2, 1)
00552 GF_MUL_8BY128(^=, 1, 2, 2)
00553 GF_MUL_8BY128(^=, 1, 2, 3)
00554 GF_MUL_8BY128(^=, 1, 3, 0)
00555 GF_MUL_8BY128(^=, 1, 3, 1)
00556 GF_MUL_8BY128(^=, 1, 3, 2)
00557 GF_MUL_8BY128(^=, 1, 3, 3)
00558
00559 x0 = a0; x1 = a1;
00560 }
00561 while (len >= HASH_BLOCKSIZE);
00562
00563 hashBuffer[0] = x0; hashBuffer[1] = x1;
00564 return len;
00565 }
00566 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00567
00568 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
00569 case 1:
00570 GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
00571 return len % 16;
00572 case 3:
00573 GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
00574 return len % 16;
00575 #endif
00576
00577 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
00578 case 1:
00579 {
00580 #ifdef __GNUC__
00581 __asm__ __volatile__
00582 (
00583 INTEL_NOPREFIX
00584 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00585 ALIGN 8
00586 GCM_AuthenticateBlocks_2K PROC FRAME
00587 rex_push_reg rsi
00588 push_reg rdi
00589 push_reg rbx
00590 .endprolog
00591 mov rsi, r8
00592 mov r11, r9
00593 #else
00594 AS2( mov WORD_REG(cx), data )
00595 AS2( mov WORD_REG(dx), len )
00596 AS2( mov WORD_REG(si), hashBuffer )
00597 AS2( shr WORD_REG(dx), 4 )
00598 #endif
00599
00600 #if CRYPTOPP_BOOL_X32
00601 AS1(push rbx)
00602 AS1(push rbp)
00603 #else
00604 AS_PUSH_IF86( bx)
00605 AS_PUSH_IF86( bp)
00606 #endif
00607
00608 #ifdef __GNUC__
00609 AS2( mov AS_REG_7, WORD_REG(di))
00610 #elif CRYPTOPP_BOOL_X86
00611 AS2( lea AS_REG_7, s_reductionTable)
00612 #endif
00613
00614 AS2( movdqa xmm0, [WORD_REG(si)] )
00615
00616 #define MUL_TABLE_0 WORD_REG(si) + 32
00617 #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
00618 #define RED_TABLE AS_REG_7
00619
00620 ASL(0)
00621 AS2( movdqu xmm4, [WORD_REG(cx)] )
00622 AS2( pxor xmm0, xmm4 )
00623
00624 AS2( movd ebx, xmm0 )
00625 AS2( mov eax, AS_HEX(f0f0f0f0) )
00626 AS2( and eax, ebx )
00627 AS2( shl ebx, 4 )
00628 AS2( and ebx, AS_HEX(f0f0f0f0) )
00629 AS2( movzx edi, ah )
00630 AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00631 AS2( movzx edi, al )
00632 AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00633 AS2( shr eax, 16 )
00634 AS2( movzx edi, ah )
00635 AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00636 AS2( movzx edi, al )
00637 AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
00638
00639 #define SSE2_MUL_32BITS(i) \
00640 AS2( psrldq xmm0, 4 )\
00641 AS2( movd eax, xmm0 )\
00642 AS2( and eax, AS_HEX(f0f0f0f0) )\
00643 AS2( movzx edi, bh )\
00644 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00645 AS2( movzx edi, bl )\
00646 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00647 AS2( shr ebx, 16 )\
00648 AS2( movzx edi, bh )\
00649 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00650 AS2( movzx edi, bl )\
00651 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
00652 AS2( movd ebx, xmm0 )\
00653 AS2( shl ebx, 4 )\
00654 AS2( and ebx, AS_HEX(f0f0f0f0) )\
00655 AS2( movzx edi, ah )\
00656 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00657 AS2( movzx edi, al )\
00658 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00659 AS2( shr eax, 16 )\
00660 AS2( movzx edi, ah )\
00661 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00662 AS2( movzx edi, al )\
00663 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
00664
00665 SSE2_MUL_32BITS(1)
00666 SSE2_MUL_32BITS(2)
00667 SSE2_MUL_32BITS(3)
00668
00669 AS2( movzx edi, bh )
00670 AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00671 AS2( movzx edi, bl )
00672 AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00673 AS2( shr ebx, 16 )
00674 AS2( movzx edi, bh )
00675 AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00676 AS2( movzx edi, bl )
00677 AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
00678
00679 AS2( movdqa xmm0, xmm3 )
00680 AS2( pslldq xmm3, 1 )
00681 AS2( pxor xmm2, xmm3 )
00682 AS2( movdqa xmm1, xmm2 )
00683 AS2( pslldq xmm2, 1 )
00684 AS2( pxor xmm5, xmm2 )
00685
00686 AS2( psrldq xmm0, 15 )
00687 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
00688 AS2( movd edi, xmm0 )
00689 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
00690 AS2( mov WORD_REG(di), xmm0 )
00691 #else
00692 AS2( movd WORD_REG(di), xmm0 )
00693 #endif
00694 AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00695 AS2( shl eax, 8 )
00696
00697 AS2( movdqa xmm0, xmm5 )
00698 AS2( pslldq xmm5, 1 )
00699 AS2( pxor xmm4, xmm5 )
00700
00701 AS2( psrldq xmm1, 15 )
00702 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
00703 AS2( movd edi, xmm1 )
00704 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
00705 AS2( mov WORD_REG(di), xmm1 )
00706 #else
00707 AS2( movd WORD_REG(di), xmm1 )
00708 #endif
00709 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00710 AS2( shl eax, 8 )
00711
00712 AS2( psrldq xmm0, 15 )
00713 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
00714 AS2( movd edi, xmm0 )
00715 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
00716 AS2( mov WORD_REG(di), xmm0 )
00717 #else
00718 AS2( movd WORD_REG(di), xmm0 )
00719 #endif
00720 AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
00721
00722 AS2( movd xmm0, eax )
00723 AS2( pxor xmm0, xmm4 )
00724
00725 AS2( add WORD_REG(cx), 16 )
00726 AS2( sub WORD_REG(dx), 1 )
00727 ASJ( jnz, 0, b )
00728 AS2( movdqa [WORD_REG(si)], xmm0 )
00729
00730 #if CRYPTOPP_BOOL_X32
00731 AS1(pop rbp)
00732 AS1(pop rbx)
00733 #else
00734 AS_POP_IF86( bp)
00735 AS_POP_IF86( bx)
00736 #endif
00737
00738 #ifdef __GNUC__
00739 ATT_PREFIX
00740 :
00741 : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
00742 : "memory", "cc", "%eax"
00743 #if CRYPTOPP_BOOL_X64
00744 , "%ebx", "%r11"
00745 #endif
00746 );
00747 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00748 pop rbx
00749 pop rdi
00750 pop rsi
00751 ret
00752 GCM_AuthenticateBlocks_2K ENDP
00753 #endif
00754
00755 return len%16;
00756 }
00757 case 3:
00758 {
00759 #ifdef __GNUC__
00760 __asm__ __volatile__
00761 (
00762 INTEL_NOPREFIX
00763 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00764 ALIGN 8
00765 GCM_AuthenticateBlocks_64K PROC FRAME
00766 rex_push_reg rsi
00767 push_reg rdi
00768 .endprolog
00769 mov rsi, r8
00770 #else
00771 AS2( mov WORD_REG(cx), data )
00772 AS2( mov WORD_REG(dx), len )
00773 AS2( mov WORD_REG(si), hashBuffer )
00774 AS2( shr WORD_REG(dx), 4 )
00775 #endif
00776
00777 AS2( movdqa xmm0, [WORD_REG(si)] )
00778
00779 #undef MUL_TABLE
00780 #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
00781
00782 ASL(1)
00783 AS2( movdqu xmm1, [WORD_REG(cx)] )
00784 AS2( pxor xmm1, xmm0 )
00785 AS2( pxor xmm0, xmm0 )
00786
00787 #undef SSE2_MUL_32BITS
00788 #define SSE2_MUL_32BITS(i) \
00789 AS2( movd eax, xmm1 )\
00790 AS2( psrldq xmm1, 4 )\
00791 AS2( movzx edi, al )\
00792 AS2( add WORD_REG(di), WORD_REG(di) )\
00793 AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
00794 AS2( movzx edi, ah )\
00795 AS2( add WORD_REG(di), WORD_REG(di) )\
00796 AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
00797 AS2( shr eax, 16 )\
00798 AS2( movzx edi, al )\
00799 AS2( add WORD_REG(di), WORD_REG(di) )\
00800 AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
00801 AS2( movzx edi, ah )\
00802 AS2( add WORD_REG(di), WORD_REG(di) )\
00803 AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
00804
00805 SSE2_MUL_32BITS(0)
00806 SSE2_MUL_32BITS(1)
00807 SSE2_MUL_32BITS(2)
00808 SSE2_MUL_32BITS(3)
00809
00810 AS2( add WORD_REG(cx), 16 )
00811 AS2( sub WORD_REG(dx), 1 )
00812 ASJ( jnz, 1, b )
00813 AS2( movdqa [WORD_REG(si)], xmm0 )
00814
00815 #ifdef __GNUC__
00816 ATT_PREFIX
00817 :
00818 : "c" (data), "d" (len/16), "S" (hashBuffer)
00819 : "memory", "cc", "%edi", "%eax"
00820 );
00821 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
00822 pop rdi
00823 pop rsi
00824 ret
00825 GCM_AuthenticateBlocks_64K ENDP
00826 #endif
00827
00828 return len%16;
00829 }
00830 #endif
00831 #ifndef CRYPTOPP_GENERATE_X64_MASM
00832 }
00833
00834 return len%16;
00835 }
00836
00837 void GCM_Base::AuthenticateLastHeaderBlock()
00838 {
00839 if (m_bufferedDataLength > 0)
00840 {
00841 memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
00842 m_bufferedDataLength = 0;
00843 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00844 }
00845 }
00846
00847 void GCM_Base::AuthenticateLastConfidentialBlock()
00848 {
00849 GCM_Base::AuthenticateLastHeaderBlock();
00850 PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
00851 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
00852 }
00853
00854 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
00855 {
00856 m_ctr.Seek(0);
00857 ReverseHashBufferIfNeeded();
00858 m_ctr.ProcessData(mac, HashBuffer(), macSize);
00859 }
00860
00861 NAMESPACE_END
00862
00863 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
00864 #endif