mirror of
https://github.com/shadps4-emu/ext-cryptopp.git
synced 2024-11-24 02:19:41 +00:00
39418a8512
Use PowerPC unaligned loads and stores with Power8. Formerly we were using Power7 as the floor because the IBM POWER Architecture manuals said unaligned loads and stores were available. However, some compilers generate bad code for unaligned loads and stores using `-march=power7`, so bump to a known good.
857 lines
32 KiB
C++
857 lines
32 KiB
C++
// gcm.cpp - originally written and placed in the public domain by Wei Dai.
|
|
// ARM and Aarch64 added by Jeffrey Walton. The ARM carryless
|
|
// multiply routines are less efficient because they shadow x86.
|
|
// The precomputed key table integration makes it tricky to use the
|
|
// more efficient ARMv8 implementation of the multiply and reduce.
|
|
|
|
// use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
|
|
|
|
#include "pch.h"
|
|
#include "config.h"
|
|
|
|
#ifndef CRYPTOPP_IMPORTS
|
|
#ifndef CRYPTOPP_GENERATE_X64_MASM
|
|
|
|
// Visual Studio .Net 2003 compiler crash
|
|
#if defined(_MSC_VER) && (_MSC_VER < 1400)
|
|
# pragma optimize("", off)
|
|
#endif
|
|
|
|
#include "gcm.h"
|
|
#include "cpu.h"
|
|
|
|
#if defined(CRYPTOPP_DISABLE_GCM_ASM)
|
|
# undef CRYPTOPP_X86_ASM_AVAILABLE
|
|
# undef CRYPTOPP_X32_ASM_AVAILABLE
|
|
# undef CRYPTOPP_X64_ASM_AVAILABLE
|
|
# undef CRYPTOPP_SSE2_ASM_AVAILABLE
|
|
#endif
|
|
|
|
NAMESPACE_BEGIN(CryptoPP)
|
|
|
|
#if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
|
|
// Different assemblers accept different mnemonics: 'movd eax, xmm0' vs
|
|
// 'movd rax, xmm0' vs 'mov eax, xmm0' vs 'mov rax, xmm0'
|
|
#if defined(CRYPTOPP_DISABLE_MIXED_ASM)
|
|
// 'movd eax, xmm0' only. REG_WORD() macro not used. Clang path.
|
|
# define USE_MOVD_REG32 1
|
|
#elif defined(__GNUC__) || defined(_MSC_VER)
|
|
// 'movd eax, xmm0' or 'movd rax, xmm0'. REG_WORD() macro supplies REG32 or REG64.
|
|
# define USE_MOVD_REG32_OR_REG64 1
|
|
#else
|
|
// 'mov eax, xmm0' or 'mov rax, xmm0'. REG_WORD() macro supplies REG32 or REG64.
|
|
# define USE_MOV_REG32_OR_REG64 1
|
|
#endif
|
|
#endif // CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
|
|
|
|
// Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
|
|
#define M128_CAST(x) ((__m128i *)(void *)(x))
|
|
#define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
|
|
|
|
word16 GCM_Base::s_reductionTable[256];
|
|
volatile bool GCM_Base::s_reductionTableInitialized = false;
|
|
|
|
void GCM_Base::GCTR::IncrementCounterBy256()
|
|
{
|
|
IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
|
|
}
|
|
|
|
static inline void Xor16(byte *a, const byte *b, const byte *c)
|
|
{
|
|
CRYPTOPP_ASSERT(IsAlignedOn(a,GetAlignmentOf<word64>()));
|
|
CRYPTOPP_ASSERT(IsAlignedOn(b,GetAlignmentOf<word64>()));
|
|
CRYPTOPP_ASSERT(IsAlignedOn(c,GetAlignmentOf<word64>()));
|
|
((word64 *)(void *)a)[0] = ((word64 *)(void *)b)[0] ^ ((word64 *)(void *)c)[0];
|
|
((word64 *)(void *)a)[1] = ((word64 *)(void *)b)[1] ^ ((word64 *)(void *)c)[1];
|
|
}
|
|
|
|
#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
|
|
// SunCC 5.10-5.11 compiler crash. Move GCM_Xor16_SSE2 out-of-line, and place in
|
|
// a source file with a SSE architecture switch. Also see GH #226 and GH #284.
|
|
extern void GCM_Xor16_SSE2(byte *a, const byte *b, const byte *c);
|
|
#endif // SSE2
|
|
|
|
#if CRYPTOPP_ARM_NEON_AVAILABLE
|
|
extern void GCM_Xor16_NEON(byte *a, const byte *b, const byte *c);
|
|
#endif
|
|
|
|
#if CRYPTOPP_POWER8_AVAILABLE
|
|
extern void GCM_Xor16_POWER8(byte *a, const byte *b, const byte *c);
|
|
#endif
|
|
|
|
#if CRYPTOPP_CLMUL_AVAILABLE
|
|
extern void GCM_SetKeyWithoutResync_CLMUL(const byte *hashKey, byte *mulTable, unsigned int tableSize);
|
|
extern size_t GCM_AuthenticateBlocks_CLMUL(const byte *data, size_t len, const byte *mtable, byte *hbuffer);
|
|
const unsigned int s_cltableSizeInBlocks = 8;
|
|
extern void GCM_ReverseHashBufferIfNeeded_CLMUL(byte *hashBuffer);
|
|
#endif // CRYPTOPP_CLMUL_AVAILABLE
|
|
|
|
#if CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
extern void GCM_SetKeyWithoutResync_PMULL(const byte *hashKey, byte *mulTable, unsigned int tableSize);
|
|
extern size_t GCM_AuthenticateBlocks_PMULL(const byte *data, size_t len, const byte *mtable, byte *hbuffer);
|
|
const unsigned int s_cltableSizeInBlocks = 8;
|
|
extern void GCM_ReverseHashBufferIfNeeded_PMULL(byte *hashBuffer);
|
|
#endif // CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
|
|
#if CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
extern void GCM_SetKeyWithoutResync_VMULL(const byte *hashKey, byte *mulTable, unsigned int tableSize);
|
|
extern size_t GCM_AuthenticateBlocks_VMULL(const byte *data, size_t len, const byte *mtable, byte *hbuffer);
|
|
const unsigned int s_cltableSizeInBlocks = 8;
|
|
extern void GCM_ReverseHashBufferIfNeeded_VMULL(byte *hashBuffer);
|
|
#endif // CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
|
|
void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs ¶ms)
|
|
{
|
|
BlockCipher &blockCipher = AccessBlockCipher();
|
|
blockCipher.SetKey(userKey, keylength, params);
|
|
|
|
// GCM is only defined for 16-byte block ciphers at the moment.
|
|
// However, variable blocksize support means we have to defer
|
|
// blocksize checks to runtime after the key is set. Also see
|
|
// https://github.com/weidai11/cryptopp/issues/408.
|
|
const unsigned int blockSize = blockCipher.BlockSize();
|
|
CRYPTOPP_ASSERT(blockSize == REQUIRED_BLOCKSIZE);
|
|
if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
|
|
throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
|
|
|
|
int tableSize, i, j, k;
|
|
|
|
#if CRYPTOPP_CLMUL_AVAILABLE
|
|
if (HasCLMUL())
|
|
{
|
|
// Avoid "parameter not used" error and suppress Coverity finding
|
|
(void)params.GetIntValue(Name::TableSize(), tableSize);
|
|
tableSize = s_cltableSizeInBlocks * blockSize;
|
|
CRYPTOPP_ASSERT(tableSize > static_cast<int>(blockSize));
|
|
}
|
|
else
|
|
#elif CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
// Avoid "parameter not used" error and suppress Coverity finding
|
|
(void)params.GetIntValue(Name::TableSize(), tableSize);
|
|
tableSize = s_cltableSizeInBlocks * blockSize;
|
|
CRYPTOPP_ASSERT(tableSize > static_cast<int>(blockSize));
|
|
}
|
|
else
|
|
#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
// Avoid "parameter not used" error and suppress Coverity finding
|
|
(void)params.GetIntValue(Name::TableSize(), tableSize);
|
|
tableSize = s_cltableSizeInBlocks * blockSize;
|
|
CRYPTOPP_ASSERT(tableSize > static_cast<int>(blockSize));
|
|
}
|
|
else
|
|
#endif
|
|
{
|
|
if (params.GetIntValue(Name::TableSize(), tableSize))
|
|
tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
|
|
else
|
|
tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
|
|
|
|
//#if defined(_MSC_VER) && (_MSC_VER < 1400)
|
|
// VC 2003 workaround: compiler generates bad code for 64K tables
|
|
//tableSize = 2*1024;
|
|
//#endif
|
|
}
|
|
|
|
m_buffer.resize(3*blockSize + tableSize);
|
|
byte *mulTable = MulTable();
|
|
byte *hashKey = HashKey();
|
|
memset(hashKey, 0, REQUIRED_BLOCKSIZE);
|
|
blockCipher.ProcessBlock(hashKey);
|
|
|
|
#if CRYPTOPP_CLMUL_AVAILABLE
|
|
if (HasCLMUL())
|
|
{
|
|
GCM_SetKeyWithoutResync_CLMUL(hashKey, mulTable, tableSize);
|
|
return;
|
|
}
|
|
#elif CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
GCM_SetKeyWithoutResync_PMULL(hashKey, mulTable, tableSize);
|
|
return;
|
|
}
|
|
#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
GCM_SetKeyWithoutResync_VMULL(hashKey, mulTable, tableSize);
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
word64 V0, V1;
|
|
typedef BlockGetAndPut<word64, BigEndian> Block;
|
|
Block::Get(hashKey)(V0)(V1);
|
|
|
|
if (tableSize == 64*1024)
|
|
{
|
|
for (i=0; i<128; i++)
|
|
{
|
|
k = i%8;
|
|
Block::Put(NULLPTR, mulTable+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
|
|
|
|
int x = (int)V1 & 1;
|
|
V1 = (V1>>1) | (V0<<63);
|
|
V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
|
|
}
|
|
|
|
for (i=0; i<16; i++)
|
|
{
|
|
memset(mulTable+i*256*16, 0, 16);
|
|
#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
|
|
if (HasSSE2())
|
|
for (j=2; j<=0x80; j*=2)
|
|
for (k=1; k<j; k++)
|
|
GCM_Xor16_SSE2(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
|
|
else
|
|
#elif CRYPTOPP_ARM_NEON_AVAILABLE
|
|
if (HasNEON())
|
|
for (j=2; j<=0x80; j*=2)
|
|
for (k=1; k<j; k++)
|
|
GCM_Xor16_NEON(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
|
|
else
|
|
#elif CRYPTOPP_POWER8_AVAILABLE
|
|
if (HasPower8())
|
|
for (j=2; j<=0x80; j*=2)
|
|
for (k=1; k<j; k++)
|
|
GCM_Xor16_POWER8(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
|
|
else
|
|
#endif
|
|
for (j=2; j<=0x80; j*=2)
|
|
for (k=1; k<j; k++)
|
|
Xor16(mulTable+i*256*16+(j+k)*16, mulTable+i*256*16+j*16, mulTable+i*256*16+k*16);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (!s_reductionTableInitialized)
|
|
{
|
|
s_reductionTable[0] = 0;
|
|
word16 x = 0x01c2;
|
|
s_reductionTable[1] = ByteReverse(x);
|
|
for (unsigned int ii=2; ii<=0x80; ii*=2)
|
|
{
|
|
x <<= 1;
|
|
s_reductionTable[ii] = ByteReverse(x);
|
|
for (unsigned int jj=1; jj<ii; jj++)
|
|
s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
|
|
}
|
|
s_reductionTableInitialized = true;
|
|
}
|
|
|
|
for (i=0; i<128-24; i++)
|
|
{
|
|
k = i%32;
|
|
if (k < 4)
|
|
Block::Put(NULLPTR, mulTable+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
|
|
else if (k < 8)
|
|
Block::Put(NULLPTR, mulTable+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
|
|
|
|
int x = (int)V1 & 1;
|
|
V1 = (V1>>1) | (V0<<63);
|
|
V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
|
|
}
|
|
|
|
for (i=0; i<4; i++)
|
|
{
|
|
memset(mulTable+i*256, 0, 16);
|
|
memset(mulTable+1024+i*256, 0, 16);
|
|
#if CRYPTOPP_SSE2_INTRIN_AVAILABLE || CRYPTOPP_SSE2_ASM_AVAILABLE
|
|
if (HasSSE2())
|
|
for (j=2; j<=8; j*=2)
|
|
for (k=1; k<j; k++)
|
|
{
|
|
GCM_Xor16_SSE2(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
|
|
GCM_Xor16_SSE2(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
|
|
}
|
|
else
|
|
#elif CRYPTOPP_ARM_NEON_AVAILABLE
|
|
if (HasNEON())
|
|
for (j=2; j<=8; j*=2)
|
|
for (k=1; k<j; k++)
|
|
{
|
|
GCM_Xor16_NEON(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
|
|
GCM_Xor16_NEON(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
|
|
}
|
|
else
|
|
#elif CRYPTOPP_POWER8_AVAILABLE
|
|
if (HasPower8())
|
|
for (j=2; j<=8; j*=2)
|
|
for (k=1; k<j; k++)
|
|
{
|
|
GCM_Xor16_POWER8(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
|
|
GCM_Xor16_POWER8(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
|
|
}
|
|
else
|
|
#endif
|
|
for (j=2; j<=8; j*=2)
|
|
for (k=1; k<j; k++)
|
|
{
|
|
Xor16(mulTable+i*256+(j+k)*16, mulTable+i*256+j*16, mulTable+i*256+k*16);
|
|
Xor16(mulTable+1024+i*256+(j+k)*16, mulTable+1024+i*256+j*16, mulTable+1024+i*256+k*16);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
inline void GCM_Base::ReverseHashBufferIfNeeded()
|
|
{
|
|
#if CRYPTOPP_CLMUL_AVAILABLE
|
|
if (HasCLMUL())
|
|
{
|
|
GCM_ReverseHashBufferIfNeeded_CLMUL(HashBuffer());
|
|
}
|
|
#elif CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
GCM_ReverseHashBufferIfNeeded_PMULL(HashBuffer());
|
|
}
|
|
#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
GCM_ReverseHashBufferIfNeeded_VMULL(HashBuffer());
|
|
}
|
|
#endif
|
|
}
|
|
|
|
void GCM_Base::Resync(const byte *iv, size_t len)
|
|
{
|
|
BlockCipher &cipher = AccessBlockCipher();
|
|
byte *hashBuffer = HashBuffer();
|
|
|
|
if (len == 12)
|
|
{
|
|
memcpy(hashBuffer, iv, len);
|
|
memset(hashBuffer+len, 0, 3);
|
|
hashBuffer[len+3] = 1;
|
|
}
|
|
else
|
|
{
|
|
size_t origLen = len;
|
|
memset(hashBuffer, 0, HASH_BLOCKSIZE);
|
|
|
|
if (len >= HASH_BLOCKSIZE)
|
|
{
|
|
len = GCM_Base::AuthenticateBlocks(iv, len);
|
|
iv += (origLen - len);
|
|
}
|
|
|
|
if (len > 0)
|
|
{
|
|
memcpy(m_buffer, iv, len);
|
|
memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
|
|
GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
|
|
}
|
|
|
|
PutBlock<word64, BigEndian, true>(NULLPTR, m_buffer)(0)(origLen*8);
|
|
GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
|
|
|
|
ReverseHashBufferIfNeeded();
|
|
}
|
|
|
|
if (m_state >= State_IVSet)
|
|
m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
|
|
else
|
|
m_ctr.SetCipherWithIV(cipher, hashBuffer);
|
|
|
|
m_ctr.Seek(HASH_BLOCKSIZE);
|
|
|
|
memset(hashBuffer, 0, HASH_BLOCKSIZE);
|
|
}
|
|
|
|
unsigned int GCM_Base::OptimalDataAlignment() const
|
|
{
|
|
return
|
|
#if CRYPTOPP_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
|
|
HasSSE2() ? 16 :
|
|
#elif CRYPTOPP_ARM_NEON_AVAILABLE
|
|
HasNEON() ? 4 :
|
|
#elif CRYPTOPP_POWER8_AVAILABLE
|
|
HasPower8() ? 16 :
|
|
#endif
|
|
GetBlockCipher().OptimalDataAlignment();
|
|
}
|
|
|
|
#if CRYPTOPP_MSC_VERSION
|
|
# pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
|
|
#endif
|
|
|
|
#endif // Not CRYPTOPP_GENERATE_X64_MASM
|
|
|
|
#ifdef CRYPTOPP_X64_MASM_AVAILABLE
|
|
extern "C" {
|
|
void GCM_AuthenticateBlocks_2K_SSE2(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
|
|
void GCM_AuthenticateBlocks_64K_SSE2(const byte *data, size_t blocks, word64 *hashBuffer);
|
|
}
|
|
#endif
|
|
|
|
#ifndef CRYPTOPP_GENERATE_X64_MASM
|
|
|
|
size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
|
|
{
|
|
#if CRYPTOPP_CLMUL_AVAILABLE
|
|
if (HasCLMUL())
|
|
{
|
|
return GCM_AuthenticateBlocks_CLMUL(data, len, MulTable(), HashBuffer());
|
|
}
|
|
#elif CRYPTOPP_ARM_PMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
return GCM_AuthenticateBlocks_PMULL(data, len, MulTable(), HashBuffer());
|
|
}
|
|
#elif CRYPTOPP_POWER8_VMULL_AVAILABLE
|
|
if (HasPMULL())
|
|
{
|
|
return GCM_AuthenticateBlocks_VMULL(data, len, MulTable(), HashBuffer());
|
|
}
|
|
#endif
|
|
|
|
typedef BlockGetAndPut<word64, NativeByteOrder> Block;
|
|
word64 *hashBuffer = (word64 *)(void *)HashBuffer();
|
|
CRYPTOPP_ASSERT(IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
|
|
|
|
switch (2*(m_buffer.size()>=64*1024)
|
|
#if CRYPTOPP_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
|
|
+ HasSSE2()
|
|
//#elif CRYPTOPP_ARM_NEON_AVAILABLE
|
|
// + HasNEON()
|
|
#endif
|
|
)
|
|
{
|
|
case 0: // non-SSE2 and 2K tables
|
|
{
|
|
byte *mulTable = MulTable();
|
|
word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
|
|
|
|
do
|
|
{
|
|
word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
|
|
Block::Get(data)(y0)(y1);
|
|
x0 ^= y0;
|
|
x1 ^= y1;
|
|
|
|
data += HASH_BLOCKSIZE;
|
|
len -= HASH_BLOCKSIZE;
|
|
|
|
#define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(mulTable+(a*1024)+(b*256)+c+d*8)
|
|
|
|
#if (CRYPTOPP_LITTLE_ENDIAN)
|
|
#if CRYPTOPP_BOOL_SLOW_WORD64
|
|
word32 z0 = (word32)x0;
|
|
word32 z1 = (word32)(x0>>32);
|
|
word32 z2 = (word32)x1;
|
|
word32 z3 = (word32)(x1>>32);
|
|
#define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
|
|
#else
|
|
#define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
|
|
#endif
|
|
#define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
|
|
#define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
|
|
#else
|
|
#define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
|
|
#define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
|
|
#define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
|
|
#endif
|
|
|
|
#define GF_MUL_32BY128(op, a, b, c) \
|
|
a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0); \
|
|
a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1); \
|
|
b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0); \
|
|
b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1); \
|
|
c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0); \
|
|
c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1); \
|
|
d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0); \
|
|
d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1); \
|
|
|
|
GF_MUL_32BY128(=, 0, 0, 0)
|
|
GF_MUL_32BY128(^=, 0, 1, 1)
|
|
GF_MUL_32BY128(^=, 1, 0, 2)
|
|
GF_MUL_32BY128(^=, 1, 1, 3)
|
|
|
|
word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
|
|
GF_SHIFT_8(d)
|
|
c0 ^= d0; c1 ^= d1;
|
|
r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
|
|
GF_SHIFT_8(c)
|
|
b0 ^= c0; b1 ^= c1;
|
|
r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
|
|
GF_SHIFT_8(b)
|
|
a0 ^= b0; a1 ^= b1;
|
|
a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
|
|
x0 = a0; x1 = a1;
|
|
}
|
|
while (len >= HASH_BLOCKSIZE);
|
|
|
|
hashBuffer[0] = x0; hashBuffer[1] = x1;
|
|
return len;
|
|
}
|
|
|
|
case 2: // non-SSE2 and 64K tables
|
|
{
|
|
byte *mulTable = MulTable();
|
|
word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
|
|
|
|
do
|
|
{
|
|
word64 y0, y1, a0, a1;
|
|
Block::Get(data)(y0)(y1);
|
|
x0 ^= y0;
|
|
x1 ^= y1;
|
|
|
|
data += HASH_BLOCKSIZE;
|
|
len -= HASH_BLOCKSIZE;
|
|
|
|
#undef READ_TABLE_WORD64_COMMON
|
|
#undef READ_TABLE_WORD64
|
|
|
|
#define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(mulTable+(a)*256*16+(c)+(d)*8)
|
|
|
|
#if (CRYPTOPP_LITTLE_ENDIAN)
|
|
#if CRYPTOPP_BOOL_SLOW_WORD64
|
|
word32 z0 = (word32)x0;
|
|
word32 z1 = (word32)(x0>>32);
|
|
word32 z2 = (word32)x1;
|
|
word32 z3 = (word32)(x1>>32);
|
|
#define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
|
|
#else
|
|
#define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
|
|
#endif
|
|
#else
|
|
#define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
|
|
#endif
|
|
|
|
#define GF_MUL_8BY128(op, b, c, d) \
|
|
a0 op READ_TABLE_WORD64(b, c, d, 0);\
|
|
a1 op READ_TABLE_WORD64(b, c, d, 1);\
|
|
|
|
GF_MUL_8BY128(=, 0, 0, 0)
|
|
GF_MUL_8BY128(^=, 0, 0, 1)
|
|
GF_MUL_8BY128(^=, 0, 0, 2)
|
|
GF_MUL_8BY128(^=, 0, 0, 3)
|
|
GF_MUL_8BY128(^=, 0, 1, 0)
|
|
GF_MUL_8BY128(^=, 0, 1, 1)
|
|
GF_MUL_8BY128(^=, 0, 1, 2)
|
|
GF_MUL_8BY128(^=, 0, 1, 3)
|
|
GF_MUL_8BY128(^=, 1, 2, 0)
|
|
GF_MUL_8BY128(^=, 1, 2, 1)
|
|
GF_MUL_8BY128(^=, 1, 2, 2)
|
|
GF_MUL_8BY128(^=, 1, 2, 3)
|
|
GF_MUL_8BY128(^=, 1, 3, 0)
|
|
GF_MUL_8BY128(^=, 1, 3, 1)
|
|
GF_MUL_8BY128(^=, 1, 3, 2)
|
|
GF_MUL_8BY128(^=, 1, 3, 3)
|
|
|
|
x0 = a0; x1 = a1;
|
|
}
|
|
while (len >= HASH_BLOCKSIZE);
|
|
|
|
hashBuffer[0] = x0; hashBuffer[1] = x1;
|
|
return len;
|
|
}
|
|
#endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
|
|
|
|
#ifdef CRYPTOPP_X64_MASM_AVAILABLE
|
|
case 1: // SSE2 and 2K tables
|
|
GCM_AuthenticateBlocks_2K_SSE2(data, len/16, hashBuffer, s_reductionTable);
|
|
return len % 16;
|
|
case 3: // SSE2 and 64K tables
|
|
GCM_AuthenticateBlocks_64K_SSE2(data, len/16, hashBuffer);
|
|
return len % 16;
|
|
#endif
|
|
|
|
#if CRYPTOPP_SSE2_ASM_AVAILABLE
|
|
case 1: // SSE2 and 2K tables
|
|
{
|
|
#ifdef __GNUC__
|
|
__asm__ __volatile__
|
|
(
|
|
INTEL_NOPREFIX
|
|
#elif defined(CRYPTOPP_GENERATE_X64_MASM)
|
|
ALIGN 8
|
|
GCM_AuthenticateBlocks_2K_SSE2 PROC FRAME
|
|
rex_push_reg rsi
|
|
push_reg rdi
|
|
push_reg rbx
|
|
.endprolog
|
|
mov rsi, r8
|
|
mov r11, r9
|
|
#else
|
|
AS2( mov WORD_REG(cx), data )
|
|
AS2( mov WORD_REG(dx), len )
|
|
AS2( mov WORD_REG(si), hashBuffer )
|
|
AS2( shr WORD_REG(dx), 4 )
|
|
#endif
|
|
|
|
#if CRYPTOPP_BOOL_X32
|
|
AS1(push rbx)
|
|
AS1(push rbp)
|
|
#else
|
|
AS_PUSH_IF86( bx)
|
|
AS_PUSH_IF86( bp)
|
|
#endif
|
|
|
|
#ifdef __GNUC__
|
|
AS2( mov AS_REG_7, WORD_REG(di))
|
|
#elif CRYPTOPP_BOOL_X86
|
|
AS2( lea AS_REG_7, s_reductionTable)
|
|
#endif
|
|
|
|
AS2( movdqa xmm0, [WORD_REG(si)] )
|
|
|
|
#define MUL_TABLE_0 WORD_REG(si) + 32
|
|
#define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
|
|
#define RED_TABLE AS_REG_7
|
|
|
|
ASL(0)
|
|
AS2( movdqu xmm4, [WORD_REG(cx)] )
|
|
AS2( pxor xmm0, xmm4 )
|
|
|
|
AS2( movd ebx, xmm0 )
|
|
AS2( mov eax, AS_HEX(f0f0f0f0) )
|
|
AS2( and eax, ebx )
|
|
AS2( shl ebx, 4 )
|
|
AS2( and ebx, AS_HEX(f0f0f0f0) )
|
|
AS2( movzx edi, ah )
|
|
AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
|
|
AS2( movzx edi, al )
|
|
AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
|
|
AS2( shr eax, 16 )
|
|
AS2( movzx edi, ah )
|
|
AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
|
|
AS2( movzx edi, al )
|
|
AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
|
|
|
|
#define SSE2_MUL_32BITS(i) \
|
|
AS2( psrldq xmm0, 4 )\
|
|
AS2( movd eax, xmm0 )\
|
|
AS2( and eax, AS_HEX(f0f0f0f0) )\
|
|
AS2( movzx edi, bh )\
|
|
AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
|
|
AS2( movzx edi, bl )\
|
|
AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
|
|
AS2( shr ebx, 16 )\
|
|
AS2( movzx edi, bh )\
|
|
AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
|
|
AS2( movzx edi, bl )\
|
|
AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
|
|
AS2( movd ebx, xmm0 )\
|
|
AS2( shl ebx, 4 )\
|
|
AS2( and ebx, AS_HEX(f0f0f0f0) )\
|
|
AS2( movzx edi, ah )\
|
|
AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
|
|
AS2( movzx edi, al )\
|
|
AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
|
|
AS2( shr eax, 16 )\
|
|
AS2( movzx edi, ah )\
|
|
AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
|
|
AS2( movzx edi, al )\
|
|
AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
|
|
|
|
SSE2_MUL_32BITS(1)
|
|
SSE2_MUL_32BITS(2)
|
|
SSE2_MUL_32BITS(3)
|
|
|
|
AS2( movzx edi, bh )
|
|
AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
|
|
AS2( movzx edi, bl )
|
|
AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
|
|
AS2( shr ebx, 16 )
|
|
AS2( movzx edi, bh )
|
|
AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
|
|
AS2( movzx edi, bl )
|
|
AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
|
|
|
|
AS2( movdqa xmm0, xmm3 )
|
|
AS2( pslldq xmm3, 1 )
|
|
AS2( pxor xmm2, xmm3 )
|
|
AS2( movdqa xmm1, xmm2 )
|
|
AS2( pslldq xmm2, 1 )
|
|
AS2( pxor xmm5, xmm2 )
|
|
|
|
AS2( psrldq xmm0, 15 )
|
|
#if USE_MOVD_REG32
|
|
AS2( movd edi, xmm0 )
|
|
#elif USE_MOV_REG32_OR_REG64
|
|
AS2( mov WORD_REG(di), xmm0 )
|
|
#else // GNU Assembler
|
|
AS2( movd WORD_REG(di), xmm0 )
|
|
#endif
|
|
AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
|
|
AS2( shl eax, 8 )
|
|
|
|
AS2( movdqa xmm0, xmm5 )
|
|
AS2( pslldq xmm5, 1 )
|
|
AS2( pxor xmm4, xmm5 )
|
|
|
|
AS2( psrldq xmm1, 15 )
|
|
#if USE_MOVD_REG32
|
|
AS2( movd edi, xmm1 )
|
|
#elif USE_MOV_REG32_OR_REG64
|
|
AS2( mov WORD_REG(di), xmm1 )
|
|
#else
|
|
AS2( movd WORD_REG(di), xmm1 )
|
|
#endif
|
|
AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
|
|
AS2( shl eax, 8 )
|
|
|
|
AS2( psrldq xmm0, 15 )
|
|
#if USE_MOVD_REG32
|
|
AS2( movd edi, xmm0 )
|
|
#elif USE_MOV_REG32_OR_REG64
|
|
AS2( mov WORD_REG(di), xmm0 )
|
|
#else
|
|
AS2( movd WORD_REG(di), xmm0 )
|
|
#endif
|
|
AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
|
|
|
|
AS2( movd xmm0, eax )
|
|
AS2( pxor xmm0, xmm4 )
|
|
|
|
AS2( add WORD_REG(cx), 16 )
|
|
AS2( sub WORD_REG(dx), 1 )
|
|
// ATT_NOPREFIX
|
|
ASJ( jnz, 0, b )
|
|
INTEL_NOPREFIX
|
|
AS2( movdqa [WORD_REG(si)], xmm0 )
|
|
|
|
#if CRYPTOPP_BOOL_X32
|
|
AS1(pop rbp)
|
|
AS1(pop rbx)
|
|
#else
|
|
AS_POP_IF86( bp)
|
|
AS_POP_IF86( bx)
|
|
#endif
|
|
|
|
#ifdef __GNUC__
|
|
ATT_PREFIX
|
|
:
|
|
: "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
|
|
: "memory", "cc", "%eax"
|
|
#if CRYPTOPP_BOOL_X64
|
|
, "%ebx", "%r11"
|
|
#endif
|
|
);
|
|
#elif defined(CRYPTOPP_GENERATE_X64_MASM)
|
|
pop rbx
|
|
pop rdi
|
|
pop rsi
|
|
ret
|
|
GCM_AuthenticateBlocks_2K_SSE2 ENDP
|
|
#endif
|
|
|
|
return len%16;
|
|
}
|
|
case 3: // SSE2 and 64K tables
|
|
{
|
|
#ifdef __GNUC__
|
|
__asm__ __volatile__
|
|
(
|
|
INTEL_NOPREFIX
|
|
#elif defined(CRYPTOPP_GENERATE_X64_MASM)
|
|
ALIGN 8
|
|
GCM_AuthenticateBlocks_64K_SSE2 PROC FRAME
|
|
rex_push_reg rsi
|
|
push_reg rdi
|
|
.endprolog
|
|
mov rsi, r8
|
|
#else
|
|
AS2( mov WORD_REG(cx), data )
|
|
AS2( mov WORD_REG(dx), len )
|
|
AS2( mov WORD_REG(si), hashBuffer )
|
|
AS2( shr WORD_REG(dx), 4 )
|
|
#endif
|
|
|
|
AS2( movdqa xmm0, [WORD_REG(si)] )
|
|
|
|
#undef MUL_TABLE
|
|
#define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
|
|
|
|
ASL(1)
|
|
AS2( movdqu xmm1, [WORD_REG(cx)] )
|
|
AS2( pxor xmm1, xmm0 )
|
|
AS2( pxor xmm0, xmm0 )
|
|
|
|
#undef SSE2_MUL_32BITS
|
|
#define SSE2_MUL_32BITS(i) \
|
|
AS2( movd eax, xmm1 )\
|
|
AS2( psrldq xmm1, 4 )\
|
|
AS2( movzx edi, al )\
|
|
AS2( add WORD_REG(di), WORD_REG(di) )\
|
|
AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
|
|
AS2( movzx edi, ah )\
|
|
AS2( add WORD_REG(di), WORD_REG(di) )\
|
|
AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
|
|
AS2( shr eax, 16 )\
|
|
AS2( movzx edi, al )\
|
|
AS2( add WORD_REG(di), WORD_REG(di) )\
|
|
AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
|
|
AS2( movzx edi, ah )\
|
|
AS2( add WORD_REG(di), WORD_REG(di) )\
|
|
AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
|
|
|
|
SSE2_MUL_32BITS(0)
|
|
SSE2_MUL_32BITS(1)
|
|
SSE2_MUL_32BITS(2)
|
|
SSE2_MUL_32BITS(3)
|
|
|
|
AS2( add WORD_REG(cx), 16 )
|
|
AS2( sub WORD_REG(dx), 1 )
|
|
// ATT_NOPREFIX
|
|
ASJ( jnz, 1, b )
|
|
INTEL_NOPREFIX
|
|
AS2( movdqa [WORD_REG(si)], xmm0 )
|
|
|
|
#ifdef __GNUC__
|
|
ATT_PREFIX
|
|
:
|
|
: "c" (data), "d" (len/16), "S" (hashBuffer)
|
|
: "memory", "cc", "%edi", "%eax"
|
|
);
|
|
#elif defined(CRYPTOPP_GENERATE_X64_MASM)
|
|
pop rdi
|
|
pop rsi
|
|
ret
|
|
GCM_AuthenticateBlocks_64K_SSE2 ENDP
|
|
#endif
|
|
|
|
return len%16;
|
|
}
|
|
#endif
|
|
#ifndef CRYPTOPP_GENERATE_X64_MASM
|
|
}
|
|
|
|
return len%16;
|
|
}
|
|
|
|
void GCM_Base::AuthenticateLastHeaderBlock()
|
|
{
|
|
if (m_bufferedDataLength > 0)
|
|
{
|
|
memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
|
|
m_bufferedDataLength = 0;
|
|
GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
|
|
}
|
|
}
|
|
|
|
void GCM_Base::AuthenticateLastConfidentialBlock()
|
|
{
|
|
GCM_Base::AuthenticateLastHeaderBlock();
|
|
PutBlock<word64, BigEndian, true>(NULLPTR, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
|
|
GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
|
|
}
|
|
|
|
void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
|
|
{
|
|
m_ctr.Seek(0);
|
|
ReverseHashBufferIfNeeded();
|
|
m_ctr.ProcessData(mac, HashBuffer(), macSize);
|
|
}
|
|
|
|
NAMESPACE_END
|
|
|
|
#endif // Not CRYPTOPP_GENERATE_X64_MASM
|
|
#endif
|