00001
00002
00003
00004
00005
00006
00007 #ifndef CRYPTOPP_CPU_H
00008 #define CRYPTOPP_CPU_H
00009
00010 #include "config.h"
00011
00012 #ifdef CRYPTOPP_GENERATE_X64_MASM
00013
00014 #define CRYPTOPP_X86_ASM_AVAILABLE
00015 #define CRYPTOPP_BOOL_X64 1
00016 #define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 1
00017 #define NAMESPACE_END
00018
00019 #else
00020
00021 # if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
00022 # include <emmintrin.h>
00023 # endif
00024
00025 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00026 #if !defined(__GNUC__) || defined(__SSSE3__) || defined(__INTEL_COMPILER)
00027 #include <tmmintrin.h>
00028 #else
00029 NAMESPACE_BEGIN(CryptoPP)
00030 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00031 _mm_shuffle_epi8 (__m128i a, __m128i b)
00032 {
00033 asm ("pshufb %1, %0" : "+x"(a) : "xm"(b));
00034 return a;
00035 }
00036 NAMESPACE_END
00037 #endif // tmmintrin.h
00038 #if !defined(__GNUC__) || defined(__SSE4_1__) || defined(__INTEL_COMPILER)
00039 #include <smmintrin.h>
00040 #else
00041 NAMESPACE_BEGIN(CryptoPP)
00042 __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00043 _mm_extract_epi32 (__m128i a, const int i)
00044 {
00045 int r;
00046 asm ("pextrd %2, %1, %0" : "=rm"(r) : "x"(a), "i"(i));
00047 return r;
00048 }
00049 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00050 _mm_insert_epi32 (__m128i a, int b, const int i)
00051 {
00052 asm ("pinsrd %2, %1, %0" : "+x"(a) : "rm"(b), "i"(i));
00053 return a;
00054 }
00055 NAMESPACE_END
00056 #endif // smmintrin.h
00057 #if !defined(__GNUC__) || (defined(__AES__) && defined(__PCLMUL__)) || defined(__INTEL_COMPILER)
00058 #include <wmmintrin.h>
00059 #else
00060 NAMESPACE_BEGIN(CryptoPP)
00061 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00062 _mm_clmulepi64_si128 (__m128i a, __m128i b, const int i)
00063 {
00064 asm ("pclmulqdq %2, %1, %0" : "+x"(a) : "xm"(b), "i"(i));
00065 return a;
00066 }
00067 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00068 _mm_aeskeygenassist_si128 (__m128i a, const int i)
00069 {
00070 __m128i r;
00071 asm ("aeskeygenassist %2, %1, %0" : "=x"(r) : "xm"(a), "i"(i));
00072 return r;
00073 }
00074 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00075 _mm_aesimc_si128 (__m128i a)
00076 {
00077 __m128i r;
00078 asm ("aesimc %1, %0" : "=x"(r) : "xm"(a));
00079 return r;
00080 }
00081 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00082 _mm_aesenc_si128 (__m128i a, __m128i b)
00083 {
00084 asm ("aesenc %1, %0" : "+x"(a) : "xm"(b));
00085 return a;
00086 }
00087 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00088 _mm_aesenclast_si128 (__m128i a, __m128i b)
00089 {
00090 asm ("aesenclast %1, %0" : "+x"(a) : "xm"(b));
00091 return a;
00092 }
00093 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00094 _mm_aesdec_si128 (__m128i a, __m128i b)
00095 {
00096 asm ("aesdec %1, %0" : "+x"(a) : "xm"(b));
00097 return a;
00098 }
00099 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
00100 _mm_aesdeclast_si128 (__m128i a, __m128i b)
00101 {
00102 asm ("aesdeclast %1, %0" : "+x"(a) : "xm"(b));
00103 return a;
00104 }
00105 NAMESPACE_END
00106 #endif // wmmintrin.h
00107 #endif // CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
00108
00109 NAMESPACE_BEGIN(CryptoPP)
00110
00111 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
00112
00113 #define CRYPTOPP_CPUID_AVAILABLE
00114
00115
00116 extern CRYPTOPP_DLL bool g_x86DetectionDone;
00117 extern CRYPTOPP_DLL bool g_hasMMX;
00118 extern CRYPTOPP_DLL bool g_hasISSE;
00119 extern CRYPTOPP_DLL bool g_hasSSE2;
00120 extern CRYPTOPP_DLL bool g_hasSSSE3;
00121 extern CRYPTOPP_DLL bool g_hasAESNI;
00122 extern CRYPTOPP_DLL bool g_hasCLMUL;
00123 extern CRYPTOPP_DLL bool g_isP4;
00124 extern CRYPTOPP_DLL bool g_hasRDRAND;
00125 extern CRYPTOPP_DLL bool g_hasRDSEED;
00126 extern CRYPTOPP_DLL word32 g_cacheLineSize;
00127
00128 CRYPTOPP_DLL void CRYPTOPP_API DetectX86Features();
00129 CRYPTOPP_DLL bool CRYPTOPP_API CpuId(word32 input, word32 output[4]);
00130
00131 inline bool HasMMX()
00132 {
00133 #if CRYPTOPP_BOOL_X64
00134 return true;
00135 #else
00136 if (!g_x86DetectionDone)
00137 DetectX86Features();
00138 return g_hasMMX;
00139 #endif
00140 }
00141
00142 inline bool HasISSE()
00143 {
00144 #if CRYPTOPP_BOOL_X64
00145 return true;
00146 #else
00147 if (!g_x86DetectionDone)
00148 DetectX86Features();
00149 return g_hasISSE;
00150 #endif
00151 }
00152
00153 inline bool HasSSE2()
00154 {
00155 #if CRYPTOPP_BOOL_X64
00156 return true;
00157 #else
00158 if (!g_x86DetectionDone)
00159 DetectX86Features();
00160 return g_hasSSE2;
00161 #endif
00162 }
00163
00164 inline bool HasSSSE3()
00165 {
00166 if (!g_x86DetectionDone)
00167 DetectX86Features();
00168 return g_hasSSSE3;
00169 }
00170
00171 inline bool HasAESNI()
00172 {
00173 if (!g_x86DetectionDone)
00174 DetectX86Features();
00175 return g_hasAESNI;
00176 }
00177
00178 inline bool HasCLMUL()
00179 {
00180 if (!g_x86DetectionDone)
00181 DetectX86Features();
00182 return g_hasCLMUL;
00183 }
00184
00185 inline bool IsP4()
00186 {
00187 if (!g_x86DetectionDone)
00188 DetectX86Features();
00189 return g_isP4;
00190 }
00191
00192 inline bool HasRDRAND()
00193 {
00194 if (!g_x86DetectionDone)
00195 DetectX86Features();
00196 return g_hasRDRAND;
00197 }
00198
00199 inline bool HasRDSEED()
00200 {
00201 if (!g_x86DetectionDone)
00202 DetectX86Features();
00203 return g_hasRDSEED;
00204 }
00205
00206 inline int GetCacheLineSize()
00207 {
00208 if (!g_x86DetectionDone)
00209 DetectX86Features();
00210 return g_cacheLineSize;
00211 }
00212
00213 #else
00214
00215 inline int GetCacheLineSize()
00216 {
00217 return CRYPTOPP_L1_CACHE_LINE_SIZE;
00218 }
00219
00220 #endif
00221
00222 #endif
00223
00224 #ifdef CRYPTOPP_GENERATE_X64_MASM
00225 #define AS1(x) x*newline*
00226 #define AS2(x, y) x, y*newline*
00227 #define AS3(x, y, z) x, y, z*newline*
00228 #define ASS(x, y, a, b, c, d) x, y, a*64+b*16+c*4+d*newline*
00229 #define ASL(x) label##x:*newline*
00230 #define ASJ(x, y, z) x label##y*newline*
00231 #define ASC(x, y) x label##y*newline*
00232 #define AS_HEX(y) 0##y##h
00233 #elif defined(_MSC_VER) || defined(__BORLANDC__)
00234 #define CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
00235 #define AS1(x) __asm {x}
00236 #define AS2(x, y) __asm {x, y}
00237 #define AS3(x, y, z) __asm {x, y, z}
00238 #define ASS(x, y, a, b, c, d) __asm {x, y, (a)*64+(b)*16+(c)*4+(d)}
00239 #define ASL(x) __asm {label##x:}
00240 #define ASJ(x, y, z) __asm {x label##y}
00241 #define ASC(x, y) __asm {x label##y}
00242 #define CRYPTOPP_NAKED __declspec(naked)
00243 #define AS_HEX(y) 0x##y
00244 #else
00245 #define CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
00246
00247 #if defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
00248 #define NEW_LINE "\n"
00249 #define INTEL_PREFIX ".intel_syntax;"
00250 #define INTEL_NOPREFIX ".intel_syntax;"
00251 #define ATT_PREFIX ".att_syntax;"
00252 #define ATT_NOPREFIX ".att_syntax;"
00253 #else
00254 #define NEW_LINE
00255 #define INTEL_PREFIX ".intel_syntax prefix;"
00256 #define INTEL_NOPREFIX ".intel_syntax noprefix;"
00257 #define ATT_PREFIX ".att_syntax prefix;"
00258 #define ATT_NOPREFIX ".att_syntax noprefix;"
00259 #endif
00260
00261
00262 #define GNU_AS1(x) #x ";" NEW_LINE
00263 #define GNU_AS2(x, y) #x ", " #y ";" NEW_LINE
00264 #define GNU_AS3(x, y, z) #x ", " #y ", " #z ";" NEW_LINE
00265 #define GNU_ASL(x) "\n" #x ":" NEW_LINE
00266 #define GNU_ASJ(x, y, z) #x " " #y #z ";" NEW_LINE
00267 #define AS1(x) GNU_AS1(x)
00268 #define AS2(x, y) GNU_AS2(x, y)
00269 #define AS3(x, y, z) GNU_AS3(x, y, z)
00270 #define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";"
00271 #define ASL(x) GNU_ASL(x)
00272 #define ASJ(x, y, z) GNU_ASJ(x, y, z)
00273 #define ASC(x, y) #x " " #y ";"
00274 #define CRYPTOPP_NAKED
00275 #define AS_HEX(y) 0x##y
00276 #endif
00277
00278 #define IF0(y)
00279 #define IF1(y) y
00280
00281 #ifdef CRYPTOPP_GENERATE_X64_MASM
00282 #define ASM_MOD(x, y) ((x) MOD (y))
00283 #define XMMWORD_PTR XMMWORD PTR
00284 #else
00285
00286 #define ASM_MOD(x, y) ((x)-((x)/(y))*(y))
00287
00288 #define XMMWORD_PTR
00289 #endif
00290
00291 #if CRYPTOPP_BOOL_X86
00292 #define AS_REG_1 ecx
00293 #define AS_REG_2 edx
00294 #define AS_REG_3 esi
00295 #define AS_REG_4 edi
00296 #define AS_REG_5 eax
00297 #define AS_REG_6 ebx
00298 #define AS_REG_7 ebp
00299 #define AS_REG_1d ecx
00300 #define AS_REG_2d edx
00301 #define AS_REG_3d esi
00302 #define AS_REG_4d edi
00303 #define AS_REG_5d eax
00304 #define AS_REG_6d ebx
00305 #define AS_REG_7d ebp
00306 #define WORD_SZ 4
00307 #define WORD_REG(x) e##x
00308 #define WORD_PTR DWORD PTR
00309 #define AS_PUSH_IF86(x) AS1(push e##x)
00310 #define AS_POP_IF86(x) AS1(pop e##x)
00311 #define AS_JCXZ jecxz
00312 #elif CRYPTOPP_BOOL_X32
00313 #define AS_REG_1 ecx
00314 #define AS_REG_2 edx
00315 #define AS_REG_3 r8d
00316 #define AS_REG_4 r9d
00317 #define AS_REG_5 eax
00318 #define AS_REG_6 r10d
00319 #define AS_REG_7 r11d
00320 #define AS_REG_1d ecx
00321 #define AS_REG_2d edx
00322 #define AS_REG_3d r8d
00323 #define AS_REG_4d r9d
00324 #define AS_REG_5d eax
00325 #define AS_REG_6d r10d
00326 #define AS_REG_7d r11d
00327 #define WORD_SZ 4
00328 #define WORD_REG(x) e##x
00329 #define WORD_PTR DWORD PTR
00330 #define AS_PUSH_IF86(x) AS1(push r##x)
00331 #define AS_POP_IF86(x) AS1(pop r##x)
00332 #define AS_JCXZ jecxz
00333 #elif CRYPTOPP_BOOL_X64
00334 #ifdef CRYPTOPP_GENERATE_X64_MASM
00335 #define AS_REG_1 rcx
00336 #define AS_REG_2 rdx
00337 #define AS_REG_3 r8
00338 #define AS_REG_4 r9
00339 #define AS_REG_5 rax
00340 #define AS_REG_6 r10
00341 #define AS_REG_7 r11
00342 #define AS_REG_1d ecx
00343 #define AS_REG_2d edx
00344 #define AS_REG_3d r8d
00345 #define AS_REG_4d r9d
00346 #define AS_REG_5d eax
00347 #define AS_REG_6d r10d
00348 #define AS_REG_7d r11d
00349 #else
00350 #define AS_REG_1 rdi
00351 #define AS_REG_2 rsi
00352 #define AS_REG_3 rdx
00353 #define AS_REG_4 rcx
00354 #define AS_REG_5 r8
00355 #define AS_REG_6 r9
00356 #define AS_REG_7 r10
00357 #define AS_REG_1d edi
00358 #define AS_REG_2d esi
00359 #define AS_REG_3d edx
00360 #define AS_REG_4d ecx
00361 #define AS_REG_5d r8d
00362 #define AS_REG_6d r9d
00363 #define AS_REG_7d r10d
00364 #endif
00365 #define WORD_SZ 8
00366 #define WORD_REG(x) r##x
00367 #define WORD_PTR QWORD PTR
00368 #define AS_PUSH_IF86(x)
00369 #define AS_POP_IF86(x)
00370 #define AS_JCXZ jrcxz
00371 #endif
00372
00373
00374 #define AS_XMM_OUTPUT4(labelPrefix, inputPtr, outputPtr, x0, x1, x2, x3, t, p0, p1, p2, p3, increment)\
00375 AS2( test inputPtr, inputPtr)\
00376 ASC( jz, labelPrefix##3)\
00377 AS2( test inputPtr, 15)\
00378 ASC( jnz, labelPrefix##7)\
00379 AS2( pxor xmm##x0, [inputPtr+p0*16])\
00380 AS2( pxor xmm##x1, [inputPtr+p1*16])\
00381 AS2( pxor xmm##x2, [inputPtr+p2*16])\
00382 AS2( pxor xmm##x3, [inputPtr+p3*16])\
00383 AS2( add inputPtr, increment*16)\
00384 ASC( jmp, labelPrefix##3)\
00385 ASL(labelPrefix##7)\
00386 AS2( movdqu xmm##t, [inputPtr+p0*16])\
00387 AS2( pxor xmm##x0, xmm##t)\
00388 AS2( movdqu xmm##t, [inputPtr+p1*16])\
00389 AS2( pxor xmm##x1, xmm##t)\
00390 AS2( movdqu xmm##t, [inputPtr+p2*16])\
00391 AS2( pxor xmm##x2, xmm##t)\
00392 AS2( movdqu xmm##t, [inputPtr+p3*16])\
00393 AS2( pxor xmm##x3, xmm##t)\
00394 AS2( add inputPtr, increment*16)\
00395 ASL(labelPrefix##3)\
00396 AS2( test outputPtr, 15)\
00397 ASC( jnz, labelPrefix##8)\
00398 AS2( movdqa [outputPtr+p0*16], xmm##x0)\
00399 AS2( movdqa [outputPtr+p1*16], xmm##x1)\
00400 AS2( movdqa [outputPtr+p2*16], xmm##x2)\
00401 AS2( movdqa [outputPtr+p3*16], xmm##x3)\
00402 ASC( jmp, labelPrefix##9)\
00403 ASL(labelPrefix##8)\
00404 AS2( movdqu [outputPtr+p0*16], xmm##x0)\
00405 AS2( movdqu [outputPtr+p1*16], xmm##x1)\
00406 AS2( movdqu [outputPtr+p2*16], xmm##x2)\
00407 AS2( movdqu [outputPtr+p3*16], xmm##x3)\
00408 ASL(labelPrefix##9)\
00409 AS2( add outputPtr, increment*16)
00410
00411 NAMESPACE_END
00412
00413 #endif