libretro/libretro-common
1
0 Fork 0
mirror of https://github.com/libretro/libretro-common.git synced 2026-11-07 11:06:12 -10:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
libretro-common/crypto/crypto.c
T
libretroadmin d9597c359a Resync
2026-10-28 19:16:45 +01:00

2937 lines
105 KiB
C
Raw Permalink Blame History

/* Copyright (C) 2026 The RetroArch team
*
* ---------------------------------------------------------------------------------------
* The following license statement only applies to this file (crypto.c).
* ---------------------------------------------------------------------------------------
*
* Permission is hereby granted, free of charge,
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include <string.h>
#include <crypto/crypto.h>
#include <retro_atomic.h>
/* AES-NI + PCLMULQDQ path for x86, chosen at run time. Same gating as
* the SHA-NI path in lrc_hash.c: GCC / Clang with the target
* attribute, so the instructions are confined to the functions that
* carry them and the TU's own baseline is untouched. */
#if (defined(__x86_64__) || defined(__i386__)) && !defined(_MSC_VER) && !defined(AES_NO_X86)
#if defined(__has_attribute) && defined(__has_include)
#if __has_attribute(target) && __has_include(<immintrin.h>) \
&& (!defined(__SCE__) || (defined(__AES__) && defined(__PCLMUL__) && defined(__SSSE3__)))
#define AES_HAVE_X86_PATH 1
#endif
#endif
#endif
#if defined(AES_HAVE_X86_PATH)
#include <immintrin.h>
#include <features/features_cpu.h>
#define AES_TARGET_X86 __attribute__((target("aes,pclmul,ssse3")))
#endif
/* ARMv8 AES + PMULL, same shape as lrc_hash.c's SHA path: used
* unconditionally when the baseline has the crypto extension, else
* compiled in under the target attribute and chosen at run time on
* toolchains whose arm_neon.h declares the intrinsics regardless of
* the baseline (Clang 16+, GCC 9+). */
#if (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_MSC_VER) && !defined(AES_NO_ARM)
#if defined(__ARM_FEATURE_CRYPTO) || (defined(__ARM_FEATURE_AES) && defined(__ARM_FEATURE_SHA2))
#define AES_HAVE_ARM_PATH 1
#elif (defined(__clang__) && __clang_major__ >= 16) \
|| (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 9)
#define AES_HAVE_ARM_PATH 1
#define AES_ARM_NEEDS_TARGET 1
#define AES_ARM_DISPATCH 1
#endif
#endif
#if defined(AES_HAVE_ARM_PATH)
#include <arm_neon.h>
#include <features/features_cpu.h>
#if defined(AES_ARM_NEEDS_TARGET)
#define AES_TARGET_ARM __attribute__((target("+crypto")))
#else
#define AES_TARGET_ARM
#endif
#endif
/* A statement for each of eight, four or one interleaved chains */
#define AES_CBC_EACH8(S) S(0) S(1) S(2) S(3) S(4) S(5) S(6) S(7)
#define AES_CBC_EACH4(S) S(0) S(1) S(2) S(3)
#define AES_CBC_EACH1(S) S(0)
int crypto_memeq_ct(const void *a, const void *b, size_t len)
{
const volatile uint8_t *pa = (const volatile uint8_t*)a;
const volatile uint8_t *pb = (const volatile uint8_t*)b;
uint8_t acc = 0;
size_t i;
for (i = 0; i < len; i++)
acc |= (uint8_t)(pa[i] ^ pb[i]);
/* 1 when every octet matched, with no data-dependent branch. */
return (int)((((unsigned)acc - 1) >> 8) & 1);
}
void crypto_memzero(void *p, size_t len)
{
volatile uint8_t *vp = (volatile uint8_t*)p;
while (len--)
*vp++ = 0;
}
static const uint8_t aes_sbox[256] = {
0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76,
0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0,
0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15,
0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75,
0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84,
0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf,
0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8,
0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2,
0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73,
0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb,
0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79,
0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08,
0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a,
0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e,
0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf,
0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16
};
static const uint8_t aes_rcon[10] = {
0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80,0x1b,0x36
};
#define XTIME(x) ((uint8_t)(((x) << 1) ^ ((((x) >> 7) & 1) * 0x1b)))
static uint32_t aes_sub_word(uint32_t w)
{
return ((uint32_t)aes_sbox[(w >> 24) & 0xff] << 24)
| ((uint32_t)aes_sbox[(w >> 16) & 0xff] << 16)
| ((uint32_t)aes_sbox[(w >> 8) & 0xff] << 8)
| (uint32_t)aes_sbox[ w & 0xff];
}
/* Round tables: ft[0][x] = (2s, s, s, 3s) with s = sbox[x], the other
* three are its byte rotations. The inverse S-box and imc[x] = (14x,
* 9x, 13x, 11x) serve the inverse cipher, which runs in the standard
* order on the encryption schedule (InvMixColumns on the state, so no
* second schedule is kept). All three are constants, generated from
* aes_sbox above: nothing to build on first use, so nothing for two
* threads setting up keys at once to race on. */
static const uint32_t aes_ft[4][256] = {
{
0xc66363a5UL,0xf87c7c84UL,0xee777799UL,0xf67b7b8dUL,0xfff2f20dUL,0xd66b6bbdUL,
0xde6f6fb1UL,0x91c5c554UL,0x60303050UL,0x02010103UL,0xce6767a9UL,0x562b2b7dUL,
0xe7fefe19UL,0xb5d7d762UL,0x4dababe6UL,0xec76769aUL,0x8fcaca45UL,0x1f82829dUL,
0x89c9c940UL,0xfa7d7d87UL,0xeffafa15UL,0xb25959ebUL,0x8e4747c9UL,0xfbf0f00bUL,
0x41adadecUL,0xb3d4d467UL,0x5fa2a2fdUL,0x45afafeaUL,0x239c9cbfUL,0x53a4a4f7UL,
0xe4727296UL,0x9bc0c05bUL,0x75b7b7c2UL,0xe1fdfd1cUL,0x3d9393aeUL,0x4c26266aUL,
0x6c36365aUL,0x7e3f3f41UL,0xf5f7f702UL,0x83cccc4fUL,0x6834345cUL,0x51a5a5f4UL,
0xd1e5e534UL,0xf9f1f108UL,0xe2717193UL,0xabd8d873UL,0x62313153UL,0x2a15153fUL,
0x0804040cUL,0x95c7c752UL,0x46232365UL,0x9dc3c35eUL,0x30181828UL,0x379696a1UL,
0x0a05050fUL,0x2f9a9ab5UL,0x0e070709UL,0x24121236UL,0x1b80809bUL,0xdfe2e23dUL,
0xcdebeb26UL,0x4e272769UL,0x7fb2b2cdUL,0xea75759fUL,0x1209091bUL,0x1d83839eUL,
0x582c2c74UL,0x341a1a2eUL,0x361b1b2dUL,0xdc6e6eb2UL,0xb45a5aeeUL,0x5ba0a0fbUL,
0xa45252f6UL,0x763b3b4dUL,0xb7d6d661UL,0x7db3b3ceUL,0x5229297bUL,0xdde3e33eUL,
0x5e2f2f71UL,0x13848497UL,0xa65353f5UL,0xb9d1d168UL,0x00000000UL,0xc1eded2cUL,
0x40202060UL,0xe3fcfc1fUL,0x79b1b1c8UL,0xb65b5bedUL,0xd46a6abeUL,0x8dcbcb46UL,
0x67bebed9UL,0x7239394bUL,0x944a4adeUL,0x984c4cd4UL,0xb05858e8UL,0x85cfcf4aUL,
0xbbd0d06bUL,0xc5efef2aUL,0x4faaaae5UL,0xedfbfb16UL,0x864343c5UL,0x9a4d4dd7UL,
0x66333355UL,0x11858594UL,0x8a4545cfUL,0xe9f9f910UL,0x04020206UL,0xfe7f7f81UL,
0xa05050f0UL,0x783c3c44UL,0x259f9fbaUL,0x4ba8a8e3UL,0xa25151f3UL,0x5da3a3feUL,
0x804040c0UL,0x058f8f8aUL,0x3f9292adUL,0x219d9dbcUL,0x70383848UL,0xf1f5f504UL,
0x63bcbcdfUL,0x77b6b6c1UL,0xafdada75UL,0x42212163UL,0x20101030UL,0xe5ffff1aUL,
0xfdf3f30eUL,0xbfd2d26dUL,0x81cdcd4cUL,0x180c0c14UL,0x26131335UL,0xc3ecec2fUL,
0xbe5f5fe1UL,0x359797a2UL,0x884444ccUL,0x2e171739UL,0x93c4c457UL,0x55a7a7f2UL,
0xfc7e7e82UL,0x7a3d3d47UL,0xc86464acUL,0xba5d5de7UL,0x3219192bUL,0xe6737395UL,
0xc06060a0UL,0x19818198UL,0x9e4f4fd1UL,0xa3dcdc7fUL,0x44222266UL,0x542a2a7eUL,
0x3b9090abUL,0x0b888883UL,0x8c4646caUL,0xc7eeee29UL,0x6bb8b8d3UL,0x2814143cUL,
0xa7dede79UL,0xbc5e5ee2UL,0x160b0b1dUL,0xaddbdb76UL,0xdbe0e03bUL,0x64323256UL,
0x743a3a4eUL,0x140a0a1eUL,0x924949dbUL,0x0c06060aUL,0x4824246cUL,0xb85c5ce4UL,
0x9fc2c25dUL,0xbdd3d36eUL,0x43acacefUL,0xc46262a6UL,0x399191a8UL,0x319595a4UL,
0xd3e4e437UL,0xf279798bUL,0xd5e7e732UL,0x8bc8c843UL,0x6e373759UL,0xda6d6db7UL,
0x018d8d8cUL,0xb1d5d564UL,0x9c4e4ed2UL,0x49a9a9e0UL,0xd86c6cb4UL,0xac5656faUL,
0xf3f4f407UL,0xcfeaea25UL,0xca6565afUL,0xf47a7a8eUL,0x47aeaee9UL,0x10080818UL,
0x6fbabad5UL,0xf0787888UL,0x4a25256fUL,0x5c2e2e72UL,0x381c1c24UL,0x57a6a6f1UL,
0x73b4b4c7UL,0x97c6c651UL,0xcbe8e823UL,0xa1dddd7cUL,0xe874749cUL,0x3e1f1f21UL,
0x964b4bddUL,0x61bdbddcUL,0x0d8b8b86UL,0x0f8a8a85UL,0xe0707090UL,0x7c3e3e42UL,
0x71b5b5c4UL,0xcc6666aaUL,0x904848d8UL,0x06030305UL,0xf7f6f601UL,0x1c0e0e12UL,
0xc26161a3UL,0x6a35355fUL,0xae5757f9UL,0x69b9b9d0UL,0x17868691UL,0x99c1c158UL,
0x3a1d1d27UL,0x279e9eb9UL,0xd9e1e138UL,0xebf8f813UL,0x2b9898b3UL,0x22111133UL,
0xd26969bbUL,0xa9d9d970UL,0x078e8e89UL,0x339494a7UL,0x2d9b9bb6UL,0x3c1e1e22UL,
0x15878792UL,0xc9e9e920UL,0x87cece49UL,0xaa5555ffUL,0x50282878UL,0xa5dfdf7aUL,
0x038c8c8fUL,0x59a1a1f8UL,0x09898980UL,0x1a0d0d17UL,0x65bfbfdaUL,0xd7e6e631UL,
0x844242c6UL,0xd06868b8UL,0x824141c3UL,0x299999b0UL,0x5a2d2d77UL,0x1e0f0f11UL,
0x7bb0b0cbUL,0xa85454fcUL,0x6dbbbbd6UL,0x2c16163aUL
},
{
0xa5c66363UL,0x84f87c7cUL,0x99ee7777UL,0x8df67b7bUL,0x0dfff2f2UL,0xbdd66b6bUL,
0xb1de6f6fUL,0x5491c5c5UL,0x50603030UL,0x03020101UL,0xa9ce6767UL,0x7d562b2bUL,
0x19e7fefeUL,0x62b5d7d7UL,0xe64dababUL,0x9aec7676UL,0x458fcacaUL,0x9d1f8282UL,
0x4089c9c9UL,0x87fa7d7dUL,0x15effafaUL,0xebb25959UL,0xc98e4747UL,0x0bfbf0f0UL,
0xec41adadUL,0x67b3d4d4UL,0xfd5fa2a2UL,0xea45afafUL,0xbf239c9cUL,0xf753a4a4UL,
0x96e47272UL,0x5b9bc0c0UL,0xc275b7b7UL,0x1ce1fdfdUL,0xae3d9393UL,0x6a4c2626UL,
0x5a6c3636UL,0x417e3f3fUL,0x02f5f7f7UL,0x4f83ccccUL,0x5c683434UL,0xf451a5a5UL,
0x34d1e5e5UL,0x08f9f1f1UL,0x93e27171UL,0x73abd8d8UL,0x53623131UL,0x3f2a1515UL,
0x0c080404UL,0x5295c7c7UL,0x65462323UL,0x5e9dc3c3UL,0x28301818UL,0xa1379696UL,
0x0f0a0505UL,0xb52f9a9aUL,0x090e0707UL,0x36241212UL,0x9b1b8080UL,0x3ddfe2e2UL,
0x26cdebebUL,0x694e2727UL,0xcd7fb2b2UL,0x9fea7575UL,0x1b120909UL,0x9e1d8383UL,
0x74582c2cUL,0x2e341a1aUL,0x2d361b1bUL,0xb2dc6e6eUL,0xeeb45a5aUL,0xfb5ba0a0UL,
0xf6a45252UL,0x4d763b3bUL,0x61b7d6d6UL,0xce7db3b3UL,0x7b522929UL,0x3edde3e3UL,
0x715e2f2fUL,0x97138484UL,0xf5a65353UL,0x68b9d1d1UL,0x00000000UL,0x2cc1ededUL,
0x60402020UL,0x1fe3fcfcUL,0xc879b1b1UL,0xedb65b5bUL,0xbed46a6aUL,0x468dcbcbUL,
0xd967bebeUL,0x4b723939UL,0xde944a4aUL,0xd4984c4cUL,0xe8b05858UL,0x4a85cfcfUL,
0x6bbbd0d0UL,0x2ac5efefUL,0xe54faaaaUL,0x16edfbfbUL,0xc5864343UL,0xd79a4d4dUL,
0x55663333UL,0x94118585UL,0xcf8a4545UL,0x10e9f9f9UL,0x06040202UL,0x81fe7f7fUL,
0xf0a05050UL,0x44783c3cUL,0xba259f9fUL,0xe34ba8a8UL,0xf3a25151UL,0xfe5da3a3UL,
0xc0804040UL,0x8a058f8fUL,0xad3f9292UL,0xbc219d9dUL,0x48703838UL,0x04f1f5f5UL,
0xdf63bcbcUL,0xc177b6b6UL,0x75afdadaUL,0x63422121UL,0x30201010UL,0x1ae5ffffUL,
0x0efdf3f3UL,0x6dbfd2d2UL,0x4c81cdcdUL,0x14180c0cUL,0x35261313UL,0x2fc3ececUL,
0xe1be5f5fUL,0xa2359797UL,0xcc884444UL,0x392e1717UL,0x5793c4c4UL,0xf255a7a7UL,
0x82fc7e7eUL,0x477a3d3dUL,0xacc86464UL,0xe7ba5d5dUL,0x2b321919UL,0x95e67373UL,
0xa0c06060UL,0x98198181UL,0xd19e4f4fUL,0x7fa3dcdcUL,0x66442222UL,0x7e542a2aUL,
0xab3b9090UL,0x830b8888UL,0xca8c4646UL,0x29c7eeeeUL,0xd36bb8b8UL,0x3c281414UL,
0x79a7dedeUL,0xe2bc5e5eUL,0x1d160b0bUL,0x76addbdbUL,0x3bdbe0e0UL,0x56643232UL,
0x4e743a3aUL,0x1e140a0aUL,0xdb924949UL,0x0a0c0606UL,0x6c482424UL,0xe4b85c5cUL,
0x5d9fc2c2UL,0x6ebdd3d3UL,0xef43acacUL,0xa6c46262UL,0xa8399191UL,0xa4319595UL,
0x37d3e4e4UL,0x8bf27979UL,0x32d5e7e7UL,0x438bc8c8UL,0x596e3737UL,0xb7da6d6dUL,
0x8c018d8dUL,0x64b1d5d5UL,0xd29c4e4eUL,0xe049a9a9UL,0xb4d86c6cUL,0xfaac5656UL,
0x07f3f4f4UL,0x25cfeaeaUL,0xafca6565UL,0x8ef47a7aUL,0xe947aeaeUL,0x18100808UL,
0xd56fbabaUL,0x88f07878UL,0x6f4a2525UL,0x725c2e2eUL,0x24381c1cUL,0xf157a6a6UL,
0xc773b4b4UL,0x5197c6c6UL,0x23cbe8e8UL,0x7ca1ddddUL,0x9ce87474UL,0x213e1f1fUL,
0xdd964b4bUL,0xdc61bdbdUL,0x860d8b8bUL,0x850f8a8aUL,0x90e07070UL,0x427c3e3eUL,
0xc471b5b5UL,0xaacc6666UL,0xd8904848UL,0x05060303UL,0x01f7f6f6UL,0x121c0e0eUL,
0xa3c26161UL,0x5f6a3535UL,0xf9ae5757UL,0xd069b9b9UL,0x91178686UL,0x5899c1c1UL,
0x273a1d1dUL,0xb9279e9eUL,0x38d9e1e1UL,0x13ebf8f8UL,0xb32b9898UL,0x33221111UL,
0xbbd26969UL,0x70a9d9d9UL,0x89078e8eUL,0xa7339494UL,0xb62d9b9bUL,0x223c1e1eUL,
0x92158787UL,0x20c9e9e9UL,0x4987ceceUL,0xffaa5555UL,0x78502828UL,0x7aa5dfdfUL,
0x8f038c8cUL,0xf859a1a1UL,0x80098989UL,0x171a0d0dUL,0xda65bfbfUL,0x31d7e6e6UL,
0xc6844242UL,0xb8d06868UL,0xc3824141UL,0xb0299999UL,0x775a2d2dUL,0x111e0f0fUL,
0xcb7bb0b0UL,0xfca85454UL,0xd66dbbbbUL,0x3a2c1616UL
},
{
0x63a5c663UL,0x7c84f87cUL,0x7799ee77UL,0x7b8df67bUL,0xf20dfff2UL,0x6bbdd66bUL,
0x6fb1de6fUL,0xc55491c5UL,0x30506030UL,0x01030201UL,0x67a9ce67UL,0x2b7d562bUL,
0xfe19e7feUL,0xd762b5d7UL,0xabe64dabUL,0x769aec76UL,0xca458fcaUL,0x829d1f82UL,
0xc94089c9UL,0x7d87fa7dUL,0xfa15effaUL,0x59ebb259UL,0x47c98e47UL,0xf00bfbf0UL,
0xadec41adUL,0xd467b3d4UL,0xa2fd5fa2UL,0xafea45afUL,0x9cbf239cUL,0xa4f753a4UL,
0x7296e472UL,0xc05b9bc0UL,0xb7c275b7UL,0xfd1ce1fdUL,0x93ae3d93UL,0x266a4c26UL,
0x365a6c36UL,0x3f417e3fUL,0xf702f5f7UL,0xcc4f83ccUL,0x345c6834UL,0xa5f451a5UL,
0xe534d1e5UL,0xf108f9f1UL,0x7193e271UL,0xd873abd8UL,0x31536231UL,0x153f2a15UL,
0x040c0804UL,0xc75295c7UL,0x23654623UL,0xc35e9dc3UL,0x18283018UL,0x96a13796UL,
0x050f0a05UL,0x9ab52f9aUL,0x07090e07UL,0x12362412UL,0x809b1b80UL,0xe23ddfe2UL,
0xeb26cdebUL,0x27694e27UL,0xb2cd7fb2UL,0x759fea75UL,0x091b1209UL,0x839e1d83UL,
0x2c74582cUL,0x1a2e341aUL,0x1b2d361bUL,0x6eb2dc6eUL,0x5aeeb45aUL,0xa0fb5ba0UL,
0x52f6a452UL,0x3b4d763bUL,0xd661b7d6UL,0xb3ce7db3UL,0x297b5229UL,0xe33edde3UL,
0x2f715e2fUL,0x84971384UL,0x53f5a653UL,0xd168b9d1UL,0x00000000UL,0xed2cc1edUL,
0x20604020UL,0xfc1fe3fcUL,0xb1c879b1UL,0x5bedb65bUL,0x6abed46aUL,0xcb468dcbUL,
0xbed967beUL,0x394b7239UL,0x4ade944aUL,0x4cd4984cUL,0x58e8b058UL,0xcf4a85cfUL,
0xd06bbbd0UL,0xef2ac5efUL,0xaae54faaUL,0xfb16edfbUL,0x43c58643UL,0x4dd79a4dUL,
0x33556633UL,0x85941185UL,0x45cf8a45UL,0xf910e9f9UL,0x02060402UL,0x7f81fe7fUL,
0x50f0a050UL,0x3c44783cUL,0x9fba259fUL,0xa8e34ba8UL,0x51f3a251UL,0xa3fe5da3UL,
0x40c08040UL,0x8f8a058fUL,0x92ad3f92UL,0x9dbc219dUL,0x38487038UL,0xf504f1f5UL,
0xbcdf63bcUL,0xb6c177b6UL,0xda75afdaUL,0x21634221UL,0x10302010UL,0xff1ae5ffUL,
0xf30efdf3UL,0xd26dbfd2UL,0xcd4c81cdUL,0x0c14180cUL,0x13352613UL,0xec2fc3ecUL,
0x5fe1be5fUL,0x97a23597UL,0x44cc8844UL,0x17392e17UL,0xc45793c4UL,0xa7f255a7UL,
0x7e82fc7eUL,0x3d477a3dUL,0x64acc864UL,0x5de7ba5dUL,0x192b3219UL,0x7395e673UL,
0x60a0c060UL,0x81981981UL,0x4fd19e4fUL,0xdc7fa3dcUL,0x22664422UL,0x2a7e542aUL,
0x90ab3b90UL,0x88830b88UL,0x46ca8c46UL,0xee29c7eeUL,0xb8d36bb8UL,0x143c2814UL,
0xde79a7deUL,0x5ee2bc5eUL,0x0b1d160bUL,0xdb76addbUL,0xe03bdbe0UL,0x32566432UL,
0x3a4e743aUL,0x0a1e140aUL,0x49db9249UL,0x060a0c06UL,0x246c4824UL,0x5ce4b85cUL,
0xc25d9fc2UL,0xd36ebdd3UL,0xacef43acUL,0x62a6c462UL,0x91a83991UL,0x95a43195UL,
0xe437d3e4UL,0x798bf279UL,0xe732d5e7UL,0xc8438bc8UL,0x37596e37UL,0x6db7da6dUL,
0x8d8c018dUL,0xd564b1d5UL,0x4ed29c4eUL,0xa9e049a9UL,0x6cb4d86cUL,0x56faac56UL,
0xf407f3f4UL,0xea25cfeaUL,0x65afca65UL,0x7a8ef47aUL,0xaee947aeUL,0x08181008UL,
0xbad56fbaUL,0x7888f078UL,0x256f4a25UL,0x2e725c2eUL,0x1c24381cUL,0xa6f157a6UL,
0xb4c773b4UL,0xc65197c6UL,0xe823cbe8UL,0xdd7ca1ddUL,0x749ce874UL,0x1f213e1fUL,
0x4bdd964bUL,0xbddc61bdUL,0x8b860d8bUL,0x8a850f8aUL,0x7090e070UL,0x3e427c3eUL,
0xb5c471b5UL,0x66aacc66UL,0x48d89048UL,0x03050603UL,0xf601f7f6UL,0x0e121c0eUL,
0x61a3c261UL,0x355f6a35UL,0x57f9ae57UL,0xb9d069b9UL,0x86911786UL,0xc15899c1UL,
0x1d273a1dUL,0x9eb9279eUL,0xe138d9e1UL,0xf813ebf8UL,0x98b32b98UL,0x11332211UL,
0x69bbd269UL,0xd970a9d9UL,0x8e89078eUL,0x94a73394UL,0x9bb62d9bUL,0x1e223c1eUL,
0x87921587UL,0xe920c9e9UL,0xce4987ceUL,0x55ffaa55UL,0x28785028UL,0xdf7aa5dfUL,
0x8c8f038cUL,0xa1f859a1UL,0x89800989UL,0x0d171a0dUL,0xbfda65bfUL,0xe631d7e6UL,
0x42c68442UL,0x68b8d068UL,0x41c38241UL,0x99b02999UL,0x2d775a2dUL,0x0f111e0fUL,
0xb0cb7bb0UL,0x54fca854UL,0xbbd66dbbUL,0x163a2c16UL
},
{
0x6363a5c6UL,0x7c7c84f8UL,0x777799eeUL,0x7b7b8df6UL,0xf2f20dffUL,0x6b6bbdd6UL,
0x6f6fb1deUL,0xc5c55491UL,0x30305060UL,0x01010302UL,0x6767a9ceUL,0x2b2b7d56UL,
0xfefe19e7UL,0xd7d762b5UL,0xababe64dUL,0x76769aecUL,0xcaca458fUL,0x82829d1fUL,
0xc9c94089UL,0x7d7d87faUL,0xfafa15efUL,0x5959ebb2UL,0x4747c98eUL,0xf0f00bfbUL,
0xadadec41UL,0xd4d467b3UL,0xa2a2fd5fUL,0xafafea45UL,0x9c9cbf23UL,0xa4a4f753UL,
0x727296e4UL,0xc0c05b9bUL,0xb7b7c275UL,0xfdfd1ce1UL,0x9393ae3dUL,0x26266a4cUL,
0x36365a6cUL,0x3f3f417eUL,0xf7f702f5UL,0xcccc4f83UL,0x34345c68UL,0xa5a5f451UL,
0xe5e534d1UL,0xf1f108f9UL,0x717193e2UL,0xd8d873abUL,0x31315362UL,0x15153f2aUL,
0x04040c08UL,0xc7c75295UL,0x23236546UL,0xc3c35e9dUL,0x18182830UL,0x9696a137UL,
0x05050f0aUL,0x9a9ab52fUL,0x0707090eUL,0x12123624UL,0x80809b1bUL,0xe2e23ddfUL,
0xebeb26cdUL,0x2727694eUL,0xb2b2cd7fUL,0x75759feaUL,0x09091b12UL,0x83839e1dUL,
0x2c2c7458UL,0x1a1a2e34UL,0x1b1b2d36UL,0x6e6eb2dcUL,0x5a5aeeb4UL,0xa0a0fb5bUL,
0x5252f6a4UL,0x3b3b4d76UL,0xd6d661b7UL,0xb3b3ce7dUL,0x29297b52UL,0xe3e33eddUL,
0x2f2f715eUL,0x84849713UL,0x5353f5a6UL,0xd1d168b9UL,0x00000000UL,0xeded2cc1UL,
0x20206040UL,0xfcfc1fe3UL,0xb1b1c879UL,0x5b5bedb6UL,0x6a6abed4UL,0xcbcb468dUL,
0xbebed967UL,0x39394b72UL,0x4a4ade94UL,0x4c4cd498UL,0x5858e8b0UL,0xcfcf4a85UL,
0xd0d06bbbUL,0xefef2ac5UL,0xaaaae54fUL,0xfbfb16edUL,0x4343c586UL,0x4d4dd79aUL,
0x33335566UL,0x85859411UL,0x4545cf8aUL,0xf9f910e9UL,0x02020604UL,0x7f7f81feUL,
0x5050f0a0UL,0x3c3c4478UL,0x9f9fba25UL,0xa8a8e34bUL,0x5151f3a2UL,0xa3a3fe5dUL,
0x4040c080UL,0x8f8f8a05UL,0x9292ad3fUL,0x9d9dbc21UL,0x38384870UL,0xf5f504f1UL,
0xbcbcdf63UL,0xb6b6c177UL,0xdada75afUL,0x21216342UL,0x10103020UL,0xffff1ae5UL,
0xf3f30efdUL,0xd2d26dbfUL,0xcdcd4c81UL,0x0c0c1418UL,0x13133526UL,0xecec2fc3UL,
0x5f5fe1beUL,0x9797a235UL,0x4444cc88UL,0x1717392eUL,0xc4c45793UL,0xa7a7f255UL,
0x7e7e82fcUL,0x3d3d477aUL,0x6464acc8UL,0x5d5de7baUL,0x19192b32UL,0x737395e6UL,
0x6060a0c0UL,0x81819819UL,0x4f4fd19eUL,0xdcdc7fa3UL,0x22226644UL,0x2a2a7e54UL,
0x9090ab3bUL,0x8888830bUL,0x4646ca8cUL,0xeeee29c7UL,0xb8b8d36bUL,0x14143c28UL,
0xdede79a7UL,0x5e5ee2bcUL,0x0b0b1d16UL,0xdbdb76adUL,0xe0e03bdbUL,0x32325664UL,
0x3a3a4e74UL,0x0a0a1e14UL,0x4949db92UL,0x06060a0cUL,0x24246c48UL,0x5c5ce4b8UL,
0xc2c25d9fUL,0xd3d36ebdUL,0xacacef43UL,0x6262a6c4UL,0x9191a839UL,0x9595a431UL,
0xe4e437d3UL,0x79798bf2UL,0xe7e732d5UL,0xc8c8438bUL,0x3737596eUL,0x6d6db7daUL,
0x8d8d8c01UL,0xd5d564b1UL,0x4e4ed29cUL,0xa9a9e049UL,0x6c6cb4d8UL,0x5656faacUL,
0xf4f407f3UL,0xeaea25cfUL,0x6565afcaUL,0x7a7a8ef4UL,0xaeaee947UL,0x08081810UL,
0xbabad56fUL,0x787888f0UL,0x25256f4aUL,0x2e2e725cUL,0x1c1c2438UL,0xa6a6f157UL,
0xb4b4c773UL,0xc6c65197UL,0xe8e823cbUL,0xdddd7ca1UL,0x74749ce8UL,0x1f1f213eUL,
0x4b4bdd96UL,0xbdbddc61UL,0x8b8b860dUL,0x8a8a850fUL,0x707090e0UL,0x3e3e427cUL,
0xb5b5c471UL,0x6666aaccUL,0x4848d890UL,0x03030506UL,0xf6f601f7UL,0x0e0e121cUL,
0x6161a3c2UL,0x35355f6aUL,0x5757f9aeUL,0xb9b9d069UL,0x86869117UL,0xc1c15899UL,
0x1d1d273aUL,0x9e9eb927UL,0xe1e138d9UL,0xf8f813ebUL,0x9898b32bUL,0x11113322UL,
0x6969bbd2UL,0xd9d970a9UL,0x8e8e8907UL,0x9494a733UL,0x9b9bb62dUL,0x1e1e223cUL,
0x87879215UL,0xe9e920c9UL,0xcece4987UL,0x5555ffaaUL,0x28287850UL,0xdfdf7aa5UL,
0x8c8c8f03UL,0xa1a1f859UL,0x89898009UL,0x0d0d171aUL,0xbfbfda65UL,0xe6e631d7UL,
0x4242c684UL,0x6868b8d0UL,0x4141c382UL,0x9999b029UL,0x2d2d775aUL,0x0f0f111eUL,
0xb0b0cb7bUL,0x5454fca8UL,0xbbbbd66dUL,0x16163a2cUL
}
};
static const uint8_t aes_isbox[256] = {
0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38,0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb,
0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87,0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb,
0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d,0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e,
0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2,0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25,
0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16,0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92,
0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda,0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84,
0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a,0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06,
0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02,0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b,
0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea,0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73,
0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85,0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e,
0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89,0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b,
0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20,0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4,
0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31,0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f,
0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d,0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef,
0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0,0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61,
0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26,0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d
};
static const uint32_t aes_imc[256] = {
0x00000000UL,0x0e090d0bUL,0x1c121a16UL,0x121b171dUL,0x3824342cUL,0x362d3927UL,
0x24362e3aUL,0x2a3f2331UL,0x70486858UL,0x7e416553UL,0x6c5a724eUL,0x62537f45UL,
0x486c5c74UL,0x4665517fUL,0x547e4662UL,0x5a774b69UL,0xe090d0b0UL,0xee99ddbbUL,
0xfc82caa6UL,0xf28bc7adUL,0xd8b4e49cUL,0xd6bde997UL,0xc4a6fe8aUL,0xcaaff381UL,
0x90d8b8e8UL,0x9ed1b5e3UL,0x8ccaa2feUL,0x82c3aff5UL,0xa8fc8cc4UL,0xa6f581cfUL,
0xb4ee96d2UL,0xbae79bd9UL,0xdb3bbb7bUL,0xd532b670UL,0xc729a16dUL,0xc920ac66UL,
0xe31f8f57UL,0xed16825cUL,0xff0d9541UL,0xf104984aUL,0xab73d323UL,0xa57ade28UL,
0xb761c935UL,0xb968c43eUL,0x9357e70fUL,0x9d5eea04UL,0x8f45fd19UL,0x814cf012UL,
0x3bab6bcbUL,0x35a266c0UL,0x27b971ddUL,0x29b07cd6UL,0x038f5fe7UL,0x0d8652ecUL,
0x1f9d45f1UL,0x119448faUL,0x4be30393UL,0x45ea0e98UL,0x57f11985UL,0x59f8148eUL,
0x73c737bfUL,0x7dce3ab4UL,0x6fd52da9UL,0x61dc20a2UL,0xad766df6UL,0xa37f60fdUL,
0xb16477e0UL,0xbf6d7aebUL,0x955259daUL,0x9b5b54d1UL,0x894043ccUL,0x87494ec7UL,
0xdd3e05aeUL,0xd33708a5UL,0xc12c1fb8UL,0xcf2512b3UL,0xe51a3182UL,0xeb133c89UL,
0xf9082b94UL,0xf701269fUL,0x4de6bd46UL,0x43efb04dUL,0x51f4a750UL,0x5ffdaa5bUL,
0x75c2896aUL,0x7bcb8461UL,0x69d0937cUL,0x67d99e77UL,0x3daed51eUL,0x33a7d815UL,
0x21bccf08UL,0x2fb5c203UL,0x058ae132UL,0x0b83ec39UL,0x1998fb24UL,0x1791f62fUL,
0x764dd68dUL,0x7844db86UL,0x6a5fcc9bUL,0x6456c190UL,0x4e69e2a1UL,0x4060efaaUL,
0x527bf8b7UL,0x5c72f5bcUL,0x0605bed5UL,0x080cb3deUL,0x1a17a4c3UL,0x141ea9c8UL,
0x3e218af9UL,0x302887f2UL,0x223390efUL,0x2c3a9de4UL,0x96dd063dUL,0x98d40b36UL,
0x8acf1c2bUL,0x84c61120UL,0xaef93211UL,0xa0f03f1aUL,0xb2eb2807UL,0xbce2250cUL,
0xe6956e65UL,0xe89c636eUL,0xfa877473UL,0xf48e7978UL,0xdeb15a49UL,0xd0b85742UL,
0xc2a3405fUL,0xccaa4d54UL,0x41ecdaf7UL,0x4fe5d7fcUL,0x5dfec0e1UL,0x53f7cdeaUL,
0x79c8eedbUL,0x77c1e3d0UL,0x65daf4cdUL,0x6bd3f9c6UL,0x31a4b2afUL,0x3fadbfa4UL,
0x2db6a8b9UL,0x23bfa5b2UL,0x09808683UL,0x07898b88UL,0x15929c95UL,0x1b9b919eUL,
0xa17c0a47UL,0xaf75074cUL,0xbd6e1051UL,0xb3671d5aUL,0x99583e6bUL,0x97513360UL,
0x854a247dUL,0x8b432976UL,0xd134621fUL,0xdf3d6f14UL,0xcd267809UL,0xc32f7502UL,
0xe9105633UL,0xe7195b38UL,0xf5024c25UL,0xfb0b412eUL,0x9ad7618cUL,0x94de6c87UL,
0x86c57b9aUL,0x88cc7691UL,0xa2f355a0UL,0xacfa58abUL,0xbee14fb6UL,0xb0e842bdUL,
0xea9f09d4UL,0xe49604dfUL,0xf68d13c2UL,0xf8841ec9UL,0xd2bb3df8UL,0xdcb230f3UL,
0xcea927eeUL,0xc0a02ae5UL,0x7a47b13cUL,0x744ebc37UL,0x6655ab2aUL,0x685ca621UL,
0x42638510UL,0x4c6a881bUL,0x5e719f06UL,0x5078920dUL,0x0a0fd964UL,0x0406d46fUL,
0x161dc372UL,0x1814ce79UL,0x322bed48UL,0x3c22e043UL,0x2e39f75eUL,0x2030fa55UL,
0xec9ab701UL,0xe293ba0aUL,0xf088ad17UL,0xfe81a01cUL,0xd4be832dUL,0xdab78e26UL,
0xc8ac993bUL,0xc6a59430UL,0x9cd2df59UL,0x92dbd252UL,0x80c0c54fUL,0x8ec9c844UL,
0xa4f6eb75UL,0xaaffe67eUL,0xb8e4f163UL,0xb6edfc68UL,0x0c0a67b1UL,0x02036abaUL,
0x10187da7UL,0x1e1170acUL,0x342e539dUL,0x3a275e96UL,0x283c498bUL,0x26354480UL,
0x7c420fe9UL,0x724b02e2UL,0x605015ffUL,0x6e5918f4UL,0x44663bc5UL,0x4a6f36ceUL,
0x587421d3UL,0x567d2cd8UL,0x37a10c7aUL,0x39a80171UL,0x2bb3166cUL,0x25ba1b67UL,
0x0f853856UL,0x018c355dUL,0x13972240UL,0x1d9e2f4bUL,0x47e96422UL,0x49e06929UL,
0x5bfb7e34UL,0x55f2733fUL,0x7fcd500eUL,0x71c45d05UL,0x63df4a18UL,0x6dd64713UL,
0xd731dccaUL,0xd938d1c1UL,0xcb23c6dcUL,0xc52acbd7UL,0xef15e8e6UL,0xe11ce5edUL,
0xf307f2f0UL,0xfd0efffbUL,0xa779b492UL,0xa970b999UL,0xbb6bae84UL,0xb562a38fUL,
0x9f5d80beUL,0x91548db5UL,0x834f9aa8UL,0x8d4697a3UL
};
#if defined(AES_HAVE_X86_PATH)
/* 0 not checked yet, 1 no, 2 yes. Threads that check at the same time
* all store the same answer; atomic, so that is not a data race. */
static retro_atomic_int_t aes_x86_ok;
static int aes_x86_available(void)
{
int v = retro_atomic_load_acquire_int(&aes_x86_ok);
if (!v)
{
uint64_t f = cpu_features_get();
v = (f & RETRO_SIMD_AES) && (f & RETRO_SIMD_PCLMUL) ? 2 : 1;
retro_atomic_store_release_int(&aes_x86_ok, v);
}
return v == 2;
}
AES_TARGET_X86
static __m128i aes_x86_bswap32(__m128i v)
{
const __m128i m = _mm_set_epi8(12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3);
return _mm_shuffle_epi8(v, m);
}
AES_TARGET_X86
static __m128i aes_x86_encrypt(const struct aes_ctx *ctx, __m128i b)
{
unsigned r;
b = _mm_xor_si128(b, aes_x86_bswap32(_mm_loadu_si128((const __m128i*)ctx->rk)));
for (r = 1; r < ctx->rounds; r++)
b = _mm_aesenc_si128(b, aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r))));
return _mm_aesenclast_si128(b, aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r))));
}
AES_TARGET_X86
static void aes_x86_encrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
_mm_storeu_si128((__m128i*)out,
aes_x86_encrypt(ctx, _mm_loadu_si128((const __m128i*)in)));
}
/* Inverse cipher on the encryption schedule: aesdec wants the round
* keys through InvMixColumns, applied here per block; a run of blocks
* goes through aes_x86_cbc_decrypt(), which makes them once. */
AES_TARGET_X86
static void aes_x86_decrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
unsigned r = ctx->rounds;
__m128i b = _mm_loadu_si128((const __m128i*)in);
b = _mm_xor_si128(b, aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r))));
for (r--; r > 0; r--)
b = _mm_aesdec_si128(b, _mm_aesimc_si128(
aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r)))));
b = _mm_aesdeclast_si128(b, aes_x86_bswap32(_mm_loadu_si128((const __m128i*)ctx->rk)));
_mm_storeu_si128((__m128i*)out, b);
}
/* CBC chains interleaved so the AES unit's latency on each is hidden
* behind the others, the round keys loaded and byte-swapped once per
* call. Eight chains, four or one, each in a register of its own, so
* no round waits on a store and a load besides the cipher. */
#define AES_X86_CBC(name, EACH) \
AES_TARGET_X86 \
static void name(const __m128i *k, unsigned rounds, uint8_t *const *iv, \
const uint8_t *const *in, uint8_t *const *out, size_t len) \
{ \
__m128i x0, x1, x2, x3, x4, x5, x6, x7; \
size_t off; \
unsigned r; \
EACH(AES_X86_CBC_LOAD) \
for (off = 0; off < len; off += 16) \
{ \
EACH(AES_X86_CBC_IN) \
for (r = 1; r < rounds; r++) \
{ \
const __m128i kr = k[r]; \
EACH(AES_X86_CBC_ROUND) \
} \
{ \
const __m128i kr = k[rounds]; \
EACH(AES_X86_CBC_LAST) \
} \
} \
EACH(AES_X86_CBC_STORE) \
(void)x0; (void)x1; (void)x2; (void)x3; \
(void)x4; (void)x5; (void)x6; (void)x7; \
}
#define AES_X86_CBC_LOAD(l) x##l = _mm_loadu_si128((const __m128i*)iv[l]);
#define AES_X86_CBC_IN(l) x##l = _mm_xor_si128(_mm_xor_si128(x##l, \
_mm_loadu_si128((const __m128i*)(in[l] + off))), k[0]);
#define AES_X86_CBC_ROUND(l) x##l = _mm_aesenc_si128(x##l, kr);
#define AES_X86_CBC_LAST(l) x##l = _mm_aesenclast_si128(x##l, kr); \
_mm_storeu_si128((__m128i*)(out[l] + off), x##l);
#define AES_X86_CBC_STORE(l) _mm_storeu_si128((__m128i*)iv[l], x##l);
AES_X86_CBC(aes_x86_cbc8, AES_CBC_EACH8)
AES_X86_CBC(aes_x86_cbc4, AES_CBC_EACH4)
AES_X86_CBC(aes_x86_cbc1, AES_CBC_EACH1)
AES_TARGET_X86
static void aes_x86_cbc_lanes(const struct aes_ctx *ctx, unsigned n,
uint8_t *const *iv, const uint8_t *const *in, uint8_t *const *out,
size_t len)
{
__m128i k[15];
unsigned r, rounds = ctx->rounds;
for (r = 0; r <= rounds; r++)
k[r] = aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r)));
for (; n >= 8; n -= 8, iv += 8, in += 8, out += 8)
aes_x86_cbc8(k, rounds, iv, in, out, len);
for (; n >= 4; n -= 4, iv += 4, in += 4, out += 4)
aes_x86_cbc4(k, rounds, iv, in, out, len);
for (; n; n--, iv++, in++, out++)
aes_x86_cbc1(k, rounds, iv, in, out, len);
}
/* CBC decryption has no chain to wait on: eight blocks go through the
* inverse cipher at once, each XORed with the ciphertext before it.
* The InvMixColumns round keys are made once per call. Every block's
* ciphertext is held in a register before its plaintext is stored, so
* @in and @out may alias. */
AES_TARGET_X86
static void aes_x86_cbc_decrypt(const struct aes_ctx *ctx, uint8_t *iv,
const uint8_t *in, uint8_t *out, size_t len)
{
__m128i k[15];
__m128i prev = _mm_loadu_si128((const __m128i*)iv);
unsigned r, nr = ctx->rounds;
k[0] = aes_x86_bswap32(_mm_loadu_si128((const __m128i*)ctx->rk));
k[nr] = aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * nr)));
for (r = 1; r < nr; r++)
k[r] = _mm_aesimc_si128(aes_x86_bswap32(
_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r))));
while (len >= 128)
{
__m128i c0 = _mm_loadu_si128((const __m128i*)(in));
__m128i c1 = _mm_loadu_si128((const __m128i*)(in + 16));
__m128i c2 = _mm_loadu_si128((const __m128i*)(in + 32));
__m128i c3 = _mm_loadu_si128((const __m128i*)(in + 48));
__m128i c4 = _mm_loadu_si128((const __m128i*)(in + 64));
__m128i c5 = _mm_loadu_si128((const __m128i*)(in + 80));
__m128i c6 = _mm_loadu_si128((const __m128i*)(in + 96));
__m128i c7 = _mm_loadu_si128((const __m128i*)(in + 112));
__m128i b0 = _mm_xor_si128(c0, k[nr]);
__m128i b1 = _mm_xor_si128(c1, k[nr]);
__m128i b2 = _mm_xor_si128(c2, k[nr]);
__m128i b3 = _mm_xor_si128(c3, k[nr]);
__m128i b4 = _mm_xor_si128(c4, k[nr]);
__m128i b5 = _mm_xor_si128(c5, k[nr]);
__m128i b6 = _mm_xor_si128(c6, k[nr]);
__m128i b7 = _mm_xor_si128(c7, k[nr]);
for (r = nr - 1; r > 0; r--)
{
__m128i kr = k[r];
b0 = _mm_aesdec_si128(b0, kr); b1 = _mm_aesdec_si128(b1, kr);
b2 = _mm_aesdec_si128(b2, kr); b3 = _mm_aesdec_si128(b3, kr);
b4 = _mm_aesdec_si128(b4, kr); b5 = _mm_aesdec_si128(b5, kr);
b6 = _mm_aesdec_si128(b6, kr); b7 = _mm_aesdec_si128(b7, kr);
}
_mm_storeu_si128((__m128i*)(out), _mm_xor_si128(_mm_aesdeclast_si128(b0, k[0]), prev));
_mm_storeu_si128((__m128i*)(out + 16), _mm_xor_si128(_mm_aesdeclast_si128(b1, k[0]), c0));
_mm_storeu_si128((__m128i*)(out + 32), _mm_xor_si128(_mm_aesdeclast_si128(b2, k[0]), c1));
_mm_storeu_si128((__m128i*)(out + 48), _mm_xor_si128(_mm_aesdeclast_si128(b3, k[0]), c2));
_mm_storeu_si128((__m128i*)(out + 64), _mm_xor_si128(_mm_aesdeclast_si128(b4, k[0]), c3));
_mm_storeu_si128((__m128i*)(out + 80), _mm_xor_si128(_mm_aesdeclast_si128(b5, k[0]), c4));
_mm_storeu_si128((__m128i*)(out + 96), _mm_xor_si128(_mm_aesdeclast_si128(b6, k[0]), c5));
_mm_storeu_si128((__m128i*)(out + 112), _mm_xor_si128(_mm_aesdeclast_si128(b7, k[0]), c6));
prev = c7;
in += 128;
out += 128;
len -= 128;
}
for (; len >= 16; in += 16, out += 16, len -= 16)
{
__m128i c = _mm_loadu_si128((const __m128i*)in);
__m128i b = _mm_xor_si128(c, k[nr]);
for (r = nr - 1; r > 0; r--)
b = _mm_aesdec_si128(b, k[r]);
_mm_storeu_si128((__m128i*)out,
_mm_xor_si128(_mm_aesdeclast_si128(b, k[0]), prev));
prev = c;
}
_mm_storeu_si128((__m128i*)iv, prev);
}
/* CTR with eight blocks in flight so the AES unit's latency is hidden,
* the round keys loaded and byte-swapped once per call rather than once
* per round. The counter block is kept byte-reversed, where its 32-bit
* big-endian counter is the low lane: one add steps it (mod 2^32, as
* GCM's inc32) and one shuffle turns it back into a block. */
AES_TARGET_X86
static void aes_x86_ctr(const struct aes_ctx *ctx, uint8_t *counter,
const uint8_t *in, uint8_t *out, size_t len)
{
const __m128i rev = _mm_set_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15);
const __m128i one = _mm_set_epi32(0, 0, 0, 1);
const __m128i two = _mm_set_epi32(0, 0, 0, 2);
const __m128i four = _mm_set_epi32(0, 0, 0, 4);
const __m128i eight = _mm_set_epi32(0, 0, 0, 8);
__m128i ctr = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)counter), rev);
__m128i rk[15];
unsigned r, nr = ctx->rounds;
for (r = 0; r <= nr; r++)
rk[r] = aes_x86_bswap32(_mm_loadu_si128((const __m128i*)(ctx->rk + 4 * r)));
while (len >= 128)
{
__m128i c1 = _mm_add_epi32(ctr, one);
__m128i c2 = _mm_add_epi32(ctr, two);
__m128i c3 = _mm_add_epi32(c1, two);
__m128i c4 = _mm_add_epi32(ctr, four);
__m128i c5 = _mm_add_epi32(c1, four);
__m128i c6 = _mm_add_epi32(c2, four);
__m128i c7 = _mm_add_epi32(c3, four);
__m128i b0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, rev), rk[0]);
__m128i b1 = _mm_xor_si128(_mm_shuffle_epi8(c1, rev), rk[0]);
__m128i b2 = _mm_xor_si128(_mm_shuffle_epi8(c2, rev), rk[0]);
__m128i b3 = _mm_xor_si128(_mm_shuffle_epi8(c3, rev), rk[0]);
__m128i b4 = _mm_xor_si128(_mm_shuffle_epi8(c4, rev), rk[0]);
__m128i b5 = _mm_xor_si128(_mm_shuffle_epi8(c5, rev), rk[0]);
__m128i b6 = _mm_xor_si128(_mm_shuffle_epi8(c6, rev), rk[0]);
__m128i b7 = _mm_xor_si128(_mm_shuffle_epi8(c7, rev), rk[0]);
for (r = 1; r < nr; r++)
{
__m128i k = rk[r];
b0 = _mm_aesenc_si128(b0, k); b1 = _mm_aesenc_si128(b1, k);
b2 = _mm_aesenc_si128(b2, k); b3 = _mm_aesenc_si128(b3, k);
b4 = _mm_aesenc_si128(b4, k); b5 = _mm_aesenc_si128(b5, k);
b6 = _mm_aesenc_si128(b6, k); b7 = _mm_aesenc_si128(b7, k);
}
_mm_storeu_si128((__m128i*)(out), _mm_xor_si128(_mm_aesenclast_si128(b0, rk[nr]), _mm_loadu_si128((const __m128i*)(in))));
_mm_storeu_si128((__m128i*)(out + 16), _mm_xor_si128(_mm_aesenclast_si128(b1, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 16))));
_mm_storeu_si128((__m128i*)(out + 32), _mm_xor_si128(_mm_aesenclast_si128(b2, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 32))));
_mm_storeu_si128((__m128i*)(out + 48), _mm_xor_si128(_mm_aesenclast_si128(b3, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 48))));
_mm_storeu_si128((__m128i*)(out + 64), _mm_xor_si128(_mm_aesenclast_si128(b4, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 64))));
_mm_storeu_si128((__m128i*)(out + 80), _mm_xor_si128(_mm_aesenclast_si128(b5, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 80))));
_mm_storeu_si128((__m128i*)(out + 96), _mm_xor_si128(_mm_aesenclast_si128(b6, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 96))));
_mm_storeu_si128((__m128i*)(out + 112), _mm_xor_si128(_mm_aesenclast_si128(b7, rk[nr]), _mm_loadu_si128((const __m128i*)(in + 112))));
ctr = _mm_add_epi32(ctr, eight);
in += 128;
out += 128;
len -= 128;
}
while (len)
{
size_t n = (len < 16) ? len : 16;
size_t i;
uint8_t ks[16];
__m128i x = _mm_xor_si128(_mm_shuffle_epi8(ctr, rev), rk[0]);
for (r = 1; r < nr; r++)
x = _mm_aesenc_si128(x, rk[r]);
x = _mm_aesenclast_si128(x, rk[nr]);
if (n == 16)
_mm_storeu_si128((__m128i*)out, _mm_xor_si128(x,
_mm_loadu_si128((const __m128i*)in)));
else
{
_mm_storeu_si128((__m128i*)ks, x);
for (i = 0; i < n; i++)
out[i] = in[i] ^ ks[i];
}
ctr = _mm_add_epi32(ctr, one);
in += n;
out += n;
len -= n;
}
_mm_storeu_si128((__m128i*)counter, _mm_shuffle_epi8(ctr, rev));
}
/* The unreduced 256-bit product of a and b, as *lo / *hi. */
AES_TARGET_X86
static void aes_x86_clmul(__m128i a, __m128i b, __m128i *lo, __m128i *hi)
{
__m128i t3 = _mm_clmulepi64_si128(a, b, 0x00);
__m128i t4 = _mm_clmulepi64_si128(a, b, 0x10);
__m128i t5 = _mm_clmulepi64_si128(a, b, 0x01);
__m128i t6 = _mm_clmulepi64_si128(a, b, 0x11);
t4 = _mm_xor_si128(t4, t5);
*lo = _mm_xor_si128(t3, _mm_slli_si128(t4, 8));
*hi = _mm_xor_si128(t6, _mm_srli_si128(t4, 8));
}
/* The product reduced: one bit left (the reflected order) and modulo
* x^128 + x^7 + x^2 + x + 1. Both steps are linear, so a sum of
* products reduces as one. */
AES_TARGET_X86
static __m128i aes_x86_reduce(__m128i t3, __m128i t6)
{
__m128i t7, t8, t9, t2, t4, t5;
t7 = _mm_srli_epi32(t3, 31);
t8 = _mm_srli_epi32(t6, 31);
t3 = _mm_slli_epi32(t3, 1);
t6 = _mm_slli_epi32(t6, 1);
t9 = _mm_srli_si128(t7, 12);
t8 = _mm_slli_si128(t8, 4);
t7 = _mm_slli_si128(t7, 4);
t3 = _mm_or_si128(t3, t7);
t6 = _mm_or_si128(t6, t8);
t6 = _mm_or_si128(t6, t9);
t7 = _mm_slli_epi32(t3, 31);
t8 = _mm_slli_epi32(t3, 30);
t9 = _mm_slli_epi32(t3, 25);
t7 = _mm_xor_si128(t7, t8);
t7 = _mm_xor_si128(t7, t9);
t8 = _mm_srli_si128(t7, 4);
t7 = _mm_slli_si128(t7, 12);
t3 = _mm_xor_si128(t3, t7);
t2 = _mm_srli_epi32(t3, 1);
t4 = _mm_srli_epi32(t3, 2);
t5 = _mm_srli_epi32(t3, 7);
t2 = _mm_xor_si128(t2, t4);
t2 = _mm_xor_si128(t2, t5);
t2 = _mm_xor_si128(t2, t8);
t3 = _mm_xor_si128(t3, t2);
return _mm_xor_si128(t6, t3);
}
/* GHASH multiply with PCLMULQDQ, operands byte-reversed so the
* reflected GCM bit order becomes a plain polynomial product
* (Intel's "Algorithm 1"). */
AES_TARGET_X86
static __m128i aes_x86_gfmul(__m128i a, __m128i b)
{
__m128i lo, hi;
aes_x86_clmul(a, b, &lo, &hi);
return aes_x86_reduce(lo, hi);
}
AES_TARGET_X86
static void aes_x86_ghash(const struct aes_gcm_ctx *ctx,
uint64_t *y_hi, uint64_t *y_lo, const uint8_t *data, size_t len)
{
const __m128i rev = _mm_set_epi8(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15);
/* h_hi / h_lo are the big-endian halves of the 16 memory octets;
* byte-reversing that whole block is what the multiply wants. */
__m128i h = _mm_shuffle_epi8(_mm_set_epi64x((long long)__builtin_bswap64(ctx->h_lo), (long long)__builtin_bswap64(ctx->h_hi)), rev);
__m128i y = _mm_shuffle_epi8(_mm_set_epi64x((long long)__builtin_bswap64(*y_lo), (long long)__builtin_bswap64(*y_hi)), rev);
uint8_t tail[16];
/* eight blocks per reduction: Y' = (Y+D0)H^8 + D1 H^7 + ... + D7 H,
* each product by Karatsuba (three multiplies, not four), the eight
* summed unreduced - low, high and middle terms apart - and folded
* and reduced once, so the multiplies overlap instead of each
* waiting on the last */
if (len >= 128)
{
__m128i hp[8], hk[8];
unsigned j;
hp[0] = h;
for (j = 1; j < 8; j++)
hp[j] = aes_x86_gfmul(hp[j - 1], h);
for (j = 0; j < 8; j++)
hk[j] = _mm_xor_si128(hp[j], _mm_srli_si128(hp[j], 8));
while (len >= 128)
{
__m128i lo = _mm_setzero_si128(), hi = lo, mid = lo;
for (j = 0; j < 8; j++)
{
__m128i d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)(data + 16 * j)), rev);
const __m128i p = hp[7 - j], pk = hk[7 - j];
if (!j)
d = _mm_xor_si128(d, y);
lo = _mm_xor_si128(lo, _mm_clmulepi64_si128(d, p, 0x00));
hi = _mm_xor_si128(hi, _mm_clmulepi64_si128(d, p, 0x11));
mid = _mm_xor_si128(mid, _mm_clmulepi64_si128(
_mm_xor_si128(d, _mm_srli_si128(d, 8)), pk, 0x00));
}
mid = _mm_xor_si128(mid, _mm_xor_si128(lo, hi));
lo = _mm_xor_si128(lo, _mm_slli_si128(mid, 8));
hi = _mm_xor_si128(hi, _mm_srli_si128(mid, 8));
y = aes_x86_reduce(lo, hi);
data += 128;
len -= 128;
}
}
while (len >= 16)
{
__m128i d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)data), rev);
y = aes_x86_gfmul(_mm_xor_si128(y, d), h);
data += 16;
len -= 16;
}
if (len)
{
memset(tail, 0, 16);
memcpy(tail, data, len);
y = aes_x86_gfmul(_mm_xor_si128(y, _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)tail), rev)), h);
}
{
uint8_t o[16];
_mm_storeu_si128((__m128i*)o, _mm_shuffle_epi8(y, rev));
*y_hi = crypto_load64_be(o);
*y_lo = crypto_load64_be(o + 8);
}
}
#endif
#if defined(AES_HAVE_ARM_PATH)
static int aes_arm_available(void)
{
#if defined(AES_ARM_DISPATCH)
static int ok = -1;
if (ok < 0)
ok = (cpu_features_get() & RETRO_SIMD_AES) ? 1 : 0;
return ok;
#else
return 1;
#endif
}
/* Round key r as the 16 memory octets: the schedule holds big-endian
* words, so a little-endian load needs each word's bytes reversed. */
AES_TARGET_ARM
static uint8x16_t aes_arm_rk(const struct aes_ctx *ctx, unsigned r)
{
return vrev32q_u8(vreinterpretq_u8_u32(vld1q_u32(ctx->rk + 4 * r)));
}
AES_TARGET_ARM
static uint8x16_t aes_arm_encrypt(const struct aes_ctx *ctx, uint8x16_t b)
{
unsigned r;
for (r = 0; r + 1 < ctx->rounds; r++)
b = vaesmcq_u8(vaeseq_u8(b, aes_arm_rk(ctx, r)));
b = vaeseq_u8(b, aes_arm_rk(ctx, r));
return veorq_u8(b, aes_arm_rk(ctx, r + 1));
}
AES_TARGET_ARM
static void aes_arm_encrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
vst1q_u8(out, aes_arm_encrypt(ctx, vld1q_u8(in)));
}
/* Equivalent inverse cipher: AESD fuses the round key add with the
* inverse shift and substitution, so the InvMixColumns that the
* standard order puts after the key add moves onto the round key. */
AES_TARGET_ARM
static void aes_arm_decrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
unsigned r = ctx->rounds;
uint8x16_t b = vaesdq_u8(vld1q_u8(in), aes_arm_rk(ctx, r));
for (r--; r > 0; r--)
b = vaesdq_u8(vaesimcq_u8(b), vaesimcq_u8(aes_arm_rk(ctx, r)));
vst1q_u8(out, veorq_u8(b, aes_arm_rk(ctx, 0)));
}
/* CBC chains interleaved, each in a register of its own, as the x86
* form above; the round keys loaded once. */
#define AES_ARM_CBC(name, EACH) \
AES_TARGET_ARM \
static void name(const uint8x16_t *k, unsigned rounds, uint8_t *const *iv, \
const uint8_t *const *in, uint8_t *const *out, size_t len) \
{ \
uint8x16_t x0, x1, x2, x3, x4, x5, x6, x7; \
size_t off; \
unsigned r; \
EACH(AES_ARM_CBC_LOAD) \
for (off = 0; off < len; off += 16) \
{ \
EACH(AES_ARM_CBC_IN) \
for (r = 0; r + 1 < rounds; r++) \
{ \
const uint8x16_t kr = k[r]; \
EACH(AES_ARM_CBC_ROUND) \
} \
{ \
const uint8x16_t kr = k[r], kl = k[r + 1]; \
EACH(AES_ARM_CBC_LAST) \
} \
} \
EACH(AES_ARM_CBC_STORE) \
(void)x0; (void)x1; (void)x2; (void)x3; \
(void)x4; (void)x5; (void)x6; (void)x7; \
}
#define AES_ARM_CBC_LOAD(l) x##l = vld1q_u8(iv[l]);
#define AES_ARM_CBC_IN(l) x##l = veorq_u8(x##l, vld1q_u8(in[l] + off));
#define AES_ARM_CBC_ROUND(l) x##l = vaesmcq_u8(vaeseq_u8(x##l, kr));
#define AES_ARM_CBC_LAST(l) x##l = veorq_u8(vaeseq_u8(x##l, kr), kl); \
vst1q_u8(out[l] + off, x##l);
#define AES_ARM_CBC_STORE(l) vst1q_u8(iv[l], x##l);
AES_ARM_CBC(aes_arm_cbc8, AES_CBC_EACH8)
AES_ARM_CBC(aes_arm_cbc4, AES_CBC_EACH4)
AES_ARM_CBC(aes_arm_cbc1, AES_CBC_EACH1)
AES_TARGET_ARM
static void aes_arm_cbc_lanes(const struct aes_ctx *ctx, unsigned n,
uint8_t *const *iv, const uint8_t *const *in, uint8_t *const *out,
size_t len)
{
uint8x16_t k[15];
unsigned r, rounds = ctx->rounds;
for (r = 0; r <= rounds; r++)
k[r] = aes_arm_rk(ctx, r);
for (; n >= 8; n -= 8, iv += 8, in += 8, out += 8)
aes_arm_cbc8(k, rounds, iv, in, out, len);
for (; n >= 4; n -= 4, iv += 4, in += 4, out += 4)
aes_arm_cbc4(k, rounds, iv, in, out, len);
for (; n; n--, iv++, in++, out++)
aes_arm_cbc1(k, rounds, iv, in, out, len);
}
/* CBC decryption, eight blocks at once as the x86 form above. AESD
* takes the round key before the inverse substitution, so a round is
* AESIMC on the state, then AESD with the round key through AESIMC. */
AES_TARGET_ARM
static void aes_arm_cbc_decrypt(const struct aes_ctx *ctx, uint8_t *iv,
const uint8_t *in, uint8_t *out, size_t len)
{
uint8x16_t k[15];
uint8x16_t prev = vld1q_u8(iv);
unsigned r, nr = ctx->rounds;
k[0] = aes_arm_rk(ctx, 0);
k[nr] = aes_arm_rk(ctx, nr);
for (r = 1; r < nr; r++)
k[r] = vaesimcq_u8(aes_arm_rk(ctx, r));
while (len >= 128)
{
uint8x16_t c0 = vld1q_u8(in);
uint8x16_t c1 = vld1q_u8(in + 16);
uint8x16_t c2 = vld1q_u8(in + 32);
uint8x16_t c3 = vld1q_u8(in + 48);
uint8x16_t c4 = vld1q_u8(in + 64);
uint8x16_t c5 = vld1q_u8(in + 80);
uint8x16_t c6 = vld1q_u8(in + 96);
uint8x16_t c7 = vld1q_u8(in + 112);
uint8x16_t b0 = vaesdq_u8(c0, k[nr]);
uint8x16_t b1 = vaesdq_u8(c1, k[nr]);
uint8x16_t b2 = vaesdq_u8(c2, k[nr]);
uint8x16_t b3 = vaesdq_u8(c3, k[nr]);
uint8x16_t b4 = vaesdq_u8(c4, k[nr]);
uint8x16_t b5 = vaesdq_u8(c5, k[nr]);
uint8x16_t b6 = vaesdq_u8(c6, k[nr]);
uint8x16_t b7 = vaesdq_u8(c7, k[nr]);
for (r = nr - 1; r > 0; r--)
{
const uint8x16_t kr = k[r];
b0 = vaesdq_u8(vaesimcq_u8(b0), kr); b1 = vaesdq_u8(vaesimcq_u8(b1), kr);
b2 = vaesdq_u8(vaesimcq_u8(b2), kr); b3 = vaesdq_u8(vaesimcq_u8(b3), kr);
b4 = vaesdq_u8(vaesimcq_u8(b4), kr); b5 = vaesdq_u8(vaesimcq_u8(b5), kr);
b6 = vaesdq_u8(vaesimcq_u8(b6), kr); b7 = vaesdq_u8(vaesimcq_u8(b7), kr);
}
vst1q_u8(out, veorq_u8(veorq_u8(b0, k[0]), prev));
vst1q_u8(out + 16, veorq_u8(veorq_u8(b1, k[0]), c0));
vst1q_u8(out + 32, veorq_u8(veorq_u8(b2, k[0]), c1));
vst1q_u8(out + 48, veorq_u8(veorq_u8(b3, k[0]), c2));
vst1q_u8(out + 64, veorq_u8(veorq_u8(b4, k[0]), c3));
vst1q_u8(out + 80, veorq_u8(veorq_u8(b5, k[0]), c4));
vst1q_u8(out + 96, veorq_u8(veorq_u8(b6, k[0]), c5));
vst1q_u8(out + 112, veorq_u8(veorq_u8(b7, k[0]), c6));
prev = c7;
in += 128;
out += 128;
len -= 128;
}
for (; len >= 16; in += 16, out += 16, len -= 16)
{
uint8x16_t c = vld1q_u8(in);
uint8x16_t b = vaesdq_u8(c, k[nr]);
for (r = nr - 1; r > 0; r--)
b = vaesdq_u8(vaesimcq_u8(b), k[r]);
vst1q_u8(out, veorq_u8(veorq_u8(b, k[0]), prev));
prev = c;
}
vst1q_u8(iv, prev);
}
/* CTR with eight blocks in flight, the round keys loaded once per
* call. The counter block is kept byte-reversed within each 32-bit
* lane, where its big-endian counter is lane 3 as a native word: one
* add steps it (mod 2^32, as GCM's inc32) and one vrev32 turns it back
* into a block - the other lanes reverse twice to themselves. */
AES_TARGET_ARM
static void aes_arm_ctr(const struct aes_ctx *ctx, uint8_t *counter,
const uint8_t *in, uint8_t *out, size_t len)
{
const uint32x4_t z = vdupq_n_u32(0);
const uint32x4_t one = vsetq_lane_u32(1, z, 3);
const uint32x4_t two = vsetq_lane_u32(2, z, 3);
const uint32x4_t four = vsetq_lane_u32(4, z, 3);
const uint32x4_t eight = vsetq_lane_u32(8, z, 3);
uint32x4_t cv = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(counter)));
uint8x16_t rk[15];
unsigned r, nr = ctx->rounds;
for (r = 0; r <= nr; r++)
rk[r] = aes_arm_rk(ctx, r);
while (len >= 128)
{
uint32x4_t c1 = vaddq_u32(cv, one);
uint32x4_t c2 = vaddq_u32(cv, two);
uint32x4_t c3 = vaddq_u32(c1, two);
uint8x16_t b0 = vrev32q_u8(vreinterpretq_u8_u32(cv));
uint8x16_t b1 = vrev32q_u8(vreinterpretq_u8_u32(c1));
uint8x16_t b2 = vrev32q_u8(vreinterpretq_u8_u32(c2));
uint8x16_t b3 = vrev32q_u8(vreinterpretq_u8_u32(c3));
uint8x16_t b4 = vrev32q_u8(vreinterpretq_u8_u32(vaddq_u32(cv, four)));
uint8x16_t b5 = vrev32q_u8(vreinterpretq_u8_u32(vaddq_u32(c1, four)));
uint8x16_t b6 = vrev32q_u8(vreinterpretq_u8_u32(vaddq_u32(c2, four)));
uint8x16_t b7 = vrev32q_u8(vreinterpretq_u8_u32(vaddq_u32(c3, four)));
for (r = 0; r + 1 < nr; r++)
{
uint8x16_t k = rk[r];
b0 = vaesmcq_u8(vaeseq_u8(b0, k)); b1 = vaesmcq_u8(vaeseq_u8(b1, k));
b2 = vaesmcq_u8(vaeseq_u8(b2, k)); b3 = vaesmcq_u8(vaeseq_u8(b3, k));
b4 = vaesmcq_u8(vaeseq_u8(b4, k)); b5 = vaesmcq_u8(vaeseq_u8(b5, k));
b6 = vaesmcq_u8(vaeseq_u8(b6, k)); b7 = vaesmcq_u8(vaeseq_u8(b7, k));
}
vst1q_u8(out, veorq_u8(veorq_u8(vaeseq_u8(b0, rk[nr - 1]), rk[nr]), vld1q_u8(in)));
vst1q_u8(out + 16, veorq_u8(veorq_u8(vaeseq_u8(b1, rk[nr - 1]), rk[nr]), vld1q_u8(in + 16)));
vst1q_u8(out + 32, veorq_u8(veorq_u8(vaeseq_u8(b2, rk[nr - 1]), rk[nr]), vld1q_u8(in + 32)));
vst1q_u8(out + 48, veorq_u8(veorq_u8(vaeseq_u8(b3, rk[nr - 1]), rk[nr]), vld1q_u8(in + 48)));
vst1q_u8(out + 64, veorq_u8(veorq_u8(vaeseq_u8(b4, rk[nr - 1]), rk[nr]), vld1q_u8(in + 64)));
vst1q_u8(out + 80, veorq_u8(veorq_u8(vaeseq_u8(b5, rk[nr - 1]), rk[nr]), vld1q_u8(in + 80)));
vst1q_u8(out + 96, veorq_u8(veorq_u8(vaeseq_u8(b6, rk[nr - 1]), rk[nr]), vld1q_u8(in + 96)));
vst1q_u8(out + 112, veorq_u8(veorq_u8(vaeseq_u8(b7, rk[nr - 1]), rk[nr]), vld1q_u8(in + 112)));
cv = vaddq_u32(cv, eight);
in += 128;
out += 128;
len -= 128;
}
while (len)
{
size_t n = (len < 16) ? len : 16;
size_t i;
uint8_t tmp[16];
uint8x16_t x = vrev32q_u8(vreinterpretq_u8_u32(cv));
for (r = 0; r + 1 < nr; r++)
x = vaesmcq_u8(vaeseq_u8(x, rk[r]));
x = veorq_u8(vaeseq_u8(x, rk[nr - 1]), rk[nr]);
if (n == 16)
vst1q_u8(out, veorq_u8(x, vld1q_u8(in)));
else
{
vst1q_u8(tmp, x);
for (i = 0; i < n; i++)
out[i] = in[i] ^ tmp[i];
}
cv = vaddq_u32(cv, one);
in += n;
out += n;
len -= n;
}
vst1q_u8(counter, vrev32q_u8(vreinterpretq_u8_u32(cv)));
}
/* GHASH multiply on PMULL: the same byte-reversed-operand algorithm
* as the x86 path, with the SSE lane shifts spelled as vext and the
* word shifts as vshl / vshr. */
/* The unreduced 256-bit product of a and b, as *lo / *hi. */
AES_TARGET_ARM
static void aes_arm_clmul(uint8x16_t a, uint8x16_t b, uint8x16_t *lo, uint8x16_t *hi)
{
const uint8x16_t z = vdupq_n_u8(0);
poly64x2_t pa = vreinterpretq_p64_u8(a);
poly64x2_t pb = vreinterpretq_p64_u8(b);
uint8x16_t t3 = vreinterpretq_u8_p128(vmull_p64((poly64_t)vgetq_lane_p64(pa, 0), (poly64_t)vgetq_lane_p64(pb, 0)));
uint8x16_t t4 = vreinterpretq_u8_p128(vmull_p64((poly64_t)vgetq_lane_p64(pa, 0), (poly64_t)vgetq_lane_p64(pb, 1)));
uint8x16_t t5 = vreinterpretq_u8_p128(vmull_p64((poly64_t)vgetq_lane_p64(pa, 1), (poly64_t)vgetq_lane_p64(pb, 0)));
uint8x16_t t6 = vreinterpretq_u8_p128(vmull_high_p64(pa, pb));
t4 = veorq_u8(t4, t5);
*lo = veorq_u8(t3, vextq_u8(z, t4, 8)); /* slli_si128 8 */
*hi = veorq_u8(t6, vextq_u8(t4, z, 8)); /* srli_si128 8 */
}
/* The product reduced: one bit left and modulo x^128 + x^7 + x^2 +
* x + 1, both linear, so a sum of products reduces as one. */
AES_TARGET_ARM
static uint8x16_t aes_arm_reduce(uint8x16_t t3, uint8x16_t t6)
{
const uint8x16_t z = vdupq_n_u8(0);
uint8x16_t t7, t8, t9, t2, t4, t5;
uint32x4_t w;
w = vreinterpretq_u32_u8(t3);
t7 = vreinterpretq_u8_u32(vshrq_n_u32(w, 31));
t3 = vreinterpretq_u8_u32(vshlq_n_u32(w, 1));
w = vreinterpretq_u32_u8(t6);
t8 = vreinterpretq_u8_u32(vshrq_n_u32(w, 31));
t6 = vreinterpretq_u8_u32(vshlq_n_u32(w, 1));
t9 = vextq_u8(t7, z, 12); /* srli_si128 12 */
t8 = vextq_u8(z, t8, 12); /* slli_si128 4 */
t7 = vextq_u8(z, t7, 12); /* slli_si128 4 */
t3 = vorrq_u8(t3, t7);
t6 = vorrq_u8(t6, t8);
t6 = vorrq_u8(t6, t9);
w = vreinterpretq_u32_u8(t3);
t7 = vreinterpretq_u8_u32(vshlq_n_u32(w, 31));
t8 = vreinterpretq_u8_u32(vshlq_n_u32(w, 30));
t9 = vreinterpretq_u8_u32(vshlq_n_u32(w, 25));
t7 = veorq_u8(t7, t8);
t7 = veorq_u8(t7, t9);
t8 = vextq_u8(t7, z, 4); /* srli_si128 4 */
t7 = vextq_u8(z, t7, 4); /* slli_si128 12 */
t3 = veorq_u8(t3, t7);
w = vreinterpretq_u32_u8(t3);
t2 = vreinterpretq_u8_u32(vshrq_n_u32(w, 1));
t4 = vreinterpretq_u8_u32(vshrq_n_u32(w, 2));
t5 = vreinterpretq_u8_u32(vshrq_n_u32(w, 7));
t2 = veorq_u8(t2, t4);
t2 = veorq_u8(t2, t5);
t2 = veorq_u8(t2, t8);
t3 = veorq_u8(t3, t2);
return veorq_u8(t6, t3);
}
AES_TARGET_ARM
static uint8x16_t aes_arm_gfmul(uint8x16_t a, uint8x16_t b)
{
uint8x16_t lo, hi;
aes_arm_clmul(a, b, &lo, &hi);
return aes_arm_reduce(lo, hi);
}
AES_TARGET_ARM
static void aes_arm_ghash(const struct aes_gcm_ctx *ctx,
uint64_t *y_hi, uint64_t *y_lo, const uint8_t *data, size_t len)
{
uint8_t o[16];
uint8x16_t h, y;
crypto_store64_be(o, ctx->h_hi);
crypto_store64_be(o + 8, ctx->h_lo);
h = vrev64q_u8(vextq_u8(vld1q_u8(o), vld1q_u8(o), 8)); /* full byte reverse */
crypto_store64_be(o, *y_hi);
crypto_store64_be(o + 8, *y_lo);
y = vrev64q_u8(vextq_u8(vld1q_u8(o), vld1q_u8(o), 8));
/* eight blocks per reduction, each product by Karatsuba, the low,
* high and middle terms summed apart and folded once, as on x86 */
if (len >= 128)
{
const uint8x16_t z = vdupq_n_u8(0);
uint8x16_t hp[8], hk[8];
unsigned j;
hp[0] = h;
for (j = 1; j < 8; j++)
hp[j] = aes_arm_gfmul(hp[j - 1], h);
for (j = 0; j < 8; j++)
hk[j] = veorq_u8(hp[j], vextq_u8(hp[j], hp[j], 8));
while (len >= 128)
{
uint8x16_t lo = z, hi = z, mid = z;
for (j = 0; j < 8; j++)
{
uint8x16_t d = vld1q_u8(data + 16 * j);
uint8x16_t dk;
poly64x2_t pd, pp, pdk, ppk;
d = vrev64q_u8(vextq_u8(d, d, 8));
if (!j)
d = veorq_u8(d, y);
dk = veorq_u8(d, vextq_u8(d, d, 8));
pd = vreinterpretq_p64_u8(d);
pp = vreinterpretq_p64_u8(hp[7 - j]);
pdk = vreinterpretq_p64_u8(dk);
ppk = vreinterpretq_p64_u8(hk[7 - j]);
lo = veorq_u8(lo, vreinterpretq_u8_p128(vmull_p64((poly64_t)vgetq_lane_p64(pd, 0), (poly64_t)vgetq_lane_p64(pp, 0))));
hi = veorq_u8(hi, vreinterpretq_u8_p128(vmull_high_p64(pd, pp)));
mid = veorq_u8(mid, vreinterpretq_u8_p128(vmull_p64((poly64_t)vgetq_lane_p64(pdk, 0), (poly64_t)vgetq_lane_p64(ppk, 0))));
}
mid = veorq_u8(mid, veorq_u8(lo, hi));
lo = veorq_u8(lo, vextq_u8(z, mid, 8)); /* slli_si128 8 */
hi = veorq_u8(hi, vextq_u8(mid, z, 8)); /* srli_si128 8 */
y = aes_arm_reduce(lo, hi);
data += 128;
len -= 128;
}
}
while (len >= 16)
{
uint8x16_t d = vld1q_u8(data);
d = vrev64q_u8(vextq_u8(d, d, 8));
y = aes_arm_gfmul(veorq_u8(y, d), h);
data += 16;
len -= 16;
}
if (len)
{
uint8x16_t d;
memset(o, 0, 16);
memcpy(o, data, len);
d = vld1q_u8(o);
d = vrev64q_u8(vextq_u8(d, d, 8));
y = aes_arm_gfmul(veorq_u8(y, d), h);
}
vst1q_u8(o, vrev64q_u8(vextq_u8(y, y, 8)));
*y_hi = crypto_load64_be(o);
*y_lo = crypto_load64_be(o + 8);
}
#endif
int crypto_aes_hw(void)
{
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
return 1;
#endif
#if defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
return 1;
#endif
return 0;
}
int aes_init(struct aes_ctx *ctx, const uint8_t *key, size_t key_len)
{
unsigned i;
unsigned nk;
unsigned total;
switch (key_len)
{
case 16: nk = 4; ctx->rounds = 10; break;
case 24: nk = 6; ctx->rounds = 12; break;
case 32: nk = 8; ctx->rounds = 14; break;
default:
return -1;
}
total = 4 * (ctx->rounds + 1);
for (i = 0; i < nk; i++)
ctx->rk[i] = crypto_load32_be(key + 4 * i);
for (i = nk; i < total; i++)
{
uint32_t t = ctx->rk[i - 1];
if ((i % nk) == 0)
t = aes_sub_word(crypto_rotl32(t, 8)) ^ ((uint32_t)aes_rcon[i / nk - 1] << 24);
else if (nk > 6 && (i % nk) == 4)
t = aes_sub_word(t);
ctx->rk[i] = ctx->rk[i - nk] ^ t;
}
return 0;
}
static void aes_encrypt_block_c(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
const uint32_t *rk = ctx->rk;
uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
unsigned r;
s0 = crypto_load32_be(in) ^ rk[0];
s1 = crypto_load32_be(in + 4) ^ rk[1];
s2 = crypto_load32_be(in + 8) ^ rk[2];
s3 = crypto_load32_be(in + 12) ^ rk[3];
rk += 4;
#define AES_ROUND(o0, o1, o2, o3, i0, i1, i2, i3) \
o0 = aes_ft[0][i0 >> 24] ^ aes_ft[1][(i1 >> 16) & 0xff] \
^ aes_ft[2][(i2 >> 8) & 0xff] ^ aes_ft[3][i3 & 0xff] ^ rk[0]; \
o1 = aes_ft[0][i1 >> 24] ^ aes_ft[1][(i2 >> 16) & 0xff] \
^ aes_ft[2][(i3 >> 8) & 0xff] ^ aes_ft[3][i0 & 0xff] ^ rk[1]; \
o2 = aes_ft[0][i2 >> 24] ^ aes_ft[1][(i3 >> 16) & 0xff] \
^ aes_ft[2][(i0 >> 8) & 0xff] ^ aes_ft[3][i1 & 0xff] ^ rk[2]; \
o3 = aes_ft[0][i3 >> 24] ^ aes_ft[1][(i0 >> 16) & 0xff] \
^ aes_ft[2][(i1 >> 8) & 0xff] ^ aes_ft[3][i2 & 0xff] ^ rk[3]; \
rk += 4
for (r = 1; r < ctx->rounds; r += 2)
{
AES_ROUND(t0, t1, t2, t3, s0, s1, s2, s3);
if (r + 1 == ctx->rounds)
{
s0 = t0; s1 = t1; s2 = t2; s3 = t3;
break;
}
AES_ROUND(s0, s1, s2, s3, t0, t1, t2, t3);
}
#undef AES_ROUND
/* Last round: SubBytes + ShiftRows only. */
t0 = ((uint32_t)aes_sbox[s0 >> 24] << 24) ^ ((uint32_t)aes_sbox[(s1 >> 16) & 0xff] << 16)
^ ((uint32_t)aes_sbox[(s2 >> 8) & 0xff] << 8) ^ aes_sbox[s3 & 0xff] ^ rk[0];
t1 = ((uint32_t)aes_sbox[s1 >> 24] << 24) ^ ((uint32_t)aes_sbox[(s2 >> 16) & 0xff] << 16)
^ ((uint32_t)aes_sbox[(s3 >> 8) & 0xff] << 8) ^ aes_sbox[s0 & 0xff] ^ rk[1];
t2 = ((uint32_t)aes_sbox[s2 >> 24] << 24) ^ ((uint32_t)aes_sbox[(s3 >> 16) & 0xff] << 16)
^ ((uint32_t)aes_sbox[(s0 >> 8) & 0xff] << 8) ^ aes_sbox[s1 & 0xff] ^ rk[2];
t3 = ((uint32_t)aes_sbox[s3 >> 24] << 24) ^ ((uint32_t)aes_sbox[(s0 >> 16) & 0xff] << 16)
^ ((uint32_t)aes_sbox[(s1 >> 8) & 0xff] << 8) ^ aes_sbox[s2 & 0xff] ^ rk[3];
crypto_store32_be(out, t0);
crypto_store32_be(out + 4, t1);
crypto_store32_be(out + 8, t2);
crypto_store32_be(out + 12, t3);
}
/* InvMixColumns of one column: each output byte is the matching
* byte of imc[b] rotated into place. */
static uint32_t aes_imc_col(uint32_t c)
{
uint32_t t;
t = aes_imc[(c >> 24) & 0xff];
t ^= crypto_rotl32(aes_imc[(c >> 16) & 0xff], 24);
t ^= crypto_rotl32(aes_imc[(c >> 8) & 0xff], 16);
t ^= crypto_rotl32(aes_imc[ c & 0xff], 8);
return t;
}
/* Inverse cipher, standard order: AddRoundKey with the last round key,
* then per round InvShiftRows + InvSubBytes, AddRoundKey, and
* InvMixColumns on the state (skipped for the final round). */
static void aes_decrypt_block_c(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
const uint32_t *rk = ctx->rk + 4 * ctx->rounds;
uint32_t s0, s1, s2, s3, t0, t1, t2, t3;
unsigned r;
s0 = crypto_load32_be(in) ^ rk[0];
s1 = crypto_load32_be(in + 4) ^ rk[1];
s2 = crypto_load32_be(in + 8) ^ rk[2];
s3 = crypto_load32_be(in + 12) ^ rk[3];
for (r = ctx->rounds; r > 0; r--)
{
rk -= 4;
/* InvShiftRows moves row i right by i; column j gathers byte
* row i from column (j - i) mod 4 */
t0 = ((uint32_t)aes_isbox[(s0 >> 24) & 0xff] << 24)
| ((uint32_t)aes_isbox[(s3 >> 16) & 0xff] << 16)
| ((uint32_t)aes_isbox[(s2 >> 8) & 0xff] << 8)
| (uint32_t)aes_isbox[ s1 & 0xff];
t1 = ((uint32_t)aes_isbox[(s1 >> 24) & 0xff] << 24)
| ((uint32_t)aes_isbox[(s0 >> 16) & 0xff] << 16)
| ((uint32_t)aes_isbox[(s3 >> 8) & 0xff] << 8)
| (uint32_t)aes_isbox[ s2 & 0xff];
t2 = ((uint32_t)aes_isbox[(s2 >> 24) & 0xff] << 24)
| ((uint32_t)aes_isbox[(s1 >> 16) & 0xff] << 16)
| ((uint32_t)aes_isbox[(s0 >> 8) & 0xff] << 8)
| (uint32_t)aes_isbox[ s3 & 0xff];
t3 = ((uint32_t)aes_isbox[(s3 >> 24) & 0xff] << 24)
| ((uint32_t)aes_isbox[(s2 >> 16) & 0xff] << 16)
| ((uint32_t)aes_isbox[(s1 >> 8) & 0xff] << 8)
| (uint32_t)aes_isbox[ s0 & 0xff];
t0 ^= rk[0]; t1 ^= rk[1]; t2 ^= rk[2]; t3 ^= rk[3];
if (r > 1)
{
s0 = aes_imc_col(t0); s1 = aes_imc_col(t1);
s2 = aes_imc_col(t2); s3 = aes_imc_col(t3);
}
else
{
s0 = t0; s1 = t1; s2 = t2; s3 = t3;
}
}
crypto_store32_be(out, s0);
crypto_store32_be(out + 4, s1);
crypto_store32_be(out + 8, s2);
crypto_store32_be(out + 12, s3);
}
void aes_decrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_decrypt_block(ctx, in, out);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_decrypt_block(ctx, in, out);
return;
}
#endif
aes_decrypt_block_c(ctx, in, out);
}
void aes_encrypt_block(const struct aes_ctx *ctx,
const uint8_t *in, uint8_t *out)
{
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_encrypt_block(ctx, in, out);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_encrypt_block(ctx, in, out);
return;
}
#endif
aes_encrypt_block_c(ctx, in, out);
}
static void aes_ctr_inc32(uint8_t *counter)
{
uint32_t v = crypto_load32_be(counter + 12) + 1;
crypto_store32_be(counter + 12, v);
}
void aes_ctr_crypt(const struct aes_ctx *ctx, uint8_t *counter,
const uint8_t *in, uint8_t *out, size_t len)
{
uint8_t ks[16];
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_ctr(ctx, counter, in, out, len);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_ctr(ctx, counter, in, out, len);
return;
}
#endif
while (len)
{
size_t n = (len < 16) ? len : 16;
size_t i;
aes_encrypt_block(ctx, counter, ks);
aes_ctr_inc32(counter);
for (i = 0; i < n; i++)
out[i] = in[i] ^ ks[i];
in += n;
out += n;
len -= n;
}
crypto_memzero(ks, sizeof(ks));
}
void aes_cbc_encrypt(const struct aes_ctx *ctx, uint8_t *iv,
const uint8_t *in, uint8_t *out, size_t len)
{
aes_cbc_encrypt_lanes(ctx, 1, &iv, &in, &out, len);
}
void aes_cbc_encrypt_lanes(const struct aes_ctx *ctx, unsigned n,
uint8_t *const *iv, const uint8_t *const *in, uint8_t *const *out,
size_t len)
{
unsigned l;
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_cbc_lanes(ctx, n, iv, in, out, len);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_cbc_lanes(ctx, n, iv, in, out, len);
return;
}
#endif
for (l = 0; l < n; l++)
{
size_t off;
unsigned i;
uint8_t x[16];
memcpy(x, iv[l], 16);
for (off = 0; off < len; off += 16)
{
for (i = 0; i < 16; i++)
x[i] ^= in[l][off + i];
aes_encrypt_block(ctx, x, x);
memcpy(out[l] + off, x, 16);
}
memcpy(iv[l], x, 16);
}
}
void aes_cbc_decrypt(const struct aes_ctx *ctx, uint8_t *iv,
const uint8_t *in, uint8_t *out, size_t len)
{
uint8_t prev[16], c[16];
size_t off;
unsigned i;
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_cbc_decrypt(ctx, iv, in, out, len);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_cbc_decrypt(ctx, iv, in, out, len);
return;
}
#endif
memcpy(prev, iv, 16);
for (off = 0; off + 16 <= len; off += 16)
{
memcpy(c, in + off, 16);
aes_decrypt_block_c(ctx, c, out + off);
for (i = 0; i < 16; i++)
out[off + i] ^= prev[i];
memcpy(prev, c, 16);
}
memcpy(iv, prev, 16);
}
/* Double in GF(2^128) with the CMAC polynomial. */
static void aes_cmac_dbl(uint8_t *b)
{
unsigned i;
uint8_t carry = 0;
uint8_t msb = b[0] >> 7;
for (i = 16; i-- > 0; )
{
uint8_t next = b[i] >> 7;
b[i] = (uint8_t)((b[i] << 1) | carry);
carry = next;
}
b[15] ^= (uint8_t)(0x87 & (0 - msb));
}
void aes_cmac_init(struct aes_cmac_ctx *s, const struct aes_ctx *aes)
{
s->aes = aes;
s->n = 0;
memset(s->x, 0, 16);
}
/* The last block takes a subkey, so a full block is held back until
* more of the message shows it was not the last. */
void aes_cmac_update(struct aes_cmac_ctx *s, const uint8_t *msg, size_t len)
{
unsigned i;
/* the chain in a local for the call: through @s, every octet of it
* may alias @msg and the block XOR is not vectorised */
uint8_t x[16];
if (!len)
return;
memcpy(x, s->x, 16);
if (s->n)
{
size_t take = 16 - s->n;
if (take > len)
take = len;
memcpy(s->buf + s->n, msg, take);
s->n += take;
msg += take;
len -= take;
if (!len)
return;
for (i = 0; i < 16; i++)
x[i] ^= s->buf[i];
aes_encrypt_block(s->aes, x, x);
s->n = 0;
}
while (len > 16)
{
for (i = 0; i < 16; i++)
x[i] ^= msg[i];
aes_encrypt_block(s->aes, x, x);
msg += 16;
len -= 16;
}
memcpy(s->buf, msg, len);
s->n = len;
memcpy(s->x, x, 16);
crypto_memzero(x, sizeof(x));
}
void aes_cmac_final(struct aes_cmac_ctx *s, uint8_t *mac)
{
unsigned i;
uint8_t k[16];
memset(k, 0, 16);
aes_encrypt_block(s->aes, k, k);
aes_cmac_dbl(k);
if (s->n != 16)
{
/* Partial or empty last block: pad 10* and use K2 = dbl(K1). */
aes_cmac_dbl(k);
memset(s->buf + s->n, 0, 16 - s->n);
s->buf[s->n] = 0x80;
}
for (i = 0; i < 16; i++)
s->x[i] ^= s->buf[i] ^ k[i];
aes_encrypt_block(s->aes, s->x, mac);
crypto_memzero(k, sizeof(k));
crypto_memzero(s, sizeof(*s));
}
void aes_cmac(const struct aes_ctx *ctx,
const uint8_t *msg, size_t len, uint8_t *mac)
{
unsigned i;
size_t nblocks;
uint8_t k1[16];
uint8_t x[16];
uint8_t last[16];
memset(x, 0, 16);
aes_encrypt_block(ctx, x, k1);
aes_cmac_dbl(k1);
nblocks = (len + 15) / 16;
if (nblocks == 0)
nblocks = 1;
/* Every block but the last, straight through the chain. */
while (nblocks > 1)
{
for (i = 0; i < 16; i++)
x[i] ^= msg[i];
aes_encrypt_block(ctx, x, x);
msg += 16;
len -= 16;
nblocks--;
}
if (len == 16)
memcpy(last, msg, 16);
else
{
/* Partial or empty last block: pad 10* and use K2 = dbl(K1). */
aes_cmac_dbl(k1);
memset(last, 0, 16);
memcpy(last, msg, len);
last[len] = 0x80;
}
for (i = 0; i < 16; i++)
x[i] ^= last[i] ^ k1[i];
aes_encrypt_block(ctx, x, mac);
crypto_memzero(k1, sizeof(k1));
crypto_memzero(x, sizeof(x));
crypto_memzero(last, sizeof(last));
}
/* ---- CCM (RFC 3610) ---------------------------------------------- */
/* CBC-MAC over B_0 || AAD (length-prefixed, padded) || payload
* (padded), then CTR with A_i counters; the tag is the MAC under
* counter zero. */
int aes_ccm_stream_init(struct aes_ccm_stream *s, const struct aes_ctx *ctx,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
size_t msg_len, size_t tag_len)
{
size_t L, i, off;
size_t remaining;
uint8_t *x = s->x;
if (nonce_len < 7 || nonce_len > 13 || tag_len < 4 || tag_len > 16 || (tag_len & 1))
return -1;
L = 15 - nonce_len;
/* the length must fit L octets; shifted as 64 bits, since a 32-bit
* size_t shifted by 32 (L = 4, the 11-octet nonce SMB uses) is
* undefined and comes back unshifted on MIPS */
if (L < 8 && ((uint64_t)msg_len >> (8 * L)) != 0)
return -1;
s->aes = ctx;
s->len = msg_len;
s->off = 0;
s->L = L;
s->tag_len = tag_len;
/* B_0: flags | nonce | length of the message */
x[0] = (uint8_t)((aad_len ? 0x40 : 0) | (((tag_len - 2) / 2) << 3) | (L - 1));
memcpy(x + 1, nonce, nonce_len);
remaining = msg_len;
for (i = 0; i < L; i++)
{
x[15 - i] = (uint8_t)remaining;
remaining >>= 8;
}
aes_encrypt_block(ctx, x, x);
/* AAD, length-prefixed (2 or 6 octets here; > 2^32 is not needed) */
if (aad_len)
{
uint8_t blk[16];
size_t n;
memset(blk, 0, 16);
if (aad_len < 0xff00)
{
blk[0] = (uint8_t)(aad_len >> 8);
blk[1] = (uint8_t)aad_len;
off = 2;
}
else
{
blk[0] = 0xff; blk[1] = 0xfe;
crypto_store32_be(blk + 2, (uint32_t)aad_len);
off = 6;
}
n = 16 - off;
if (n > aad_len)
n = aad_len;
memcpy(blk + off, aad, n);
for (i = 0; i < 16; i++)
x[i] ^= blk[i];
aes_encrypt_block(ctx, x, x);
aad += n;
aad_len -= n;
while (aad_len)
{
n = aad_len < 16 ? aad_len : 16;
for (i = 0; i < n; i++)
x[i] ^= aad[i];
aes_encrypt_block(ctx, x, x);
aad += n;
aad_len -= n;
}
}
/* Counter blocks: flags = L - 1, nonce, counter */
memset(s->ctr, 0, 16);
s->ctr[0] = (uint8_t)(L - 1);
memcpy(s->ctr + 1, nonce, nonce_len);
aes_encrypt_block(ctx, s->ctr, s->s0); /* S_0, for the tag */
return 0;
}
/* A piece that starts or ends inside a block carries its keystream
* and the MAC chain across in @s; the chain is run when the block
* fills or the message ends. The chain, counter and keystream are
* locals for the call - through @s every octet may alias the data and
* the block loops are not vectorised. */
static void aes_ccm_stream_run(struct aes_ccm_stream *s, int encrypt,
const uint8_t *in, uint8_t *out, size_t len)
{
uint8_t x[16], ctr[16], ks[16];
size_t i, L = s->L, off = s->off;
if (len > s->len - off)
len = s->len - off;
if (!len)
return;
memcpy(x, s->x, 16);
memcpy(ctr, s->ctr, 16);
memcpy(ks, s->ks, 16);
while (len)
{
size_t pos = off & 15;
size_t n = 16 - pos;
if (n > len)
n = len;
if (!pos)
{
size_t c = off / 16 + 1;
for (i = 0; i < L; i++)
{
ctr[15 - i] = (uint8_t)c;
c >>= 8;
}
aes_encrypt_block(s->aes, ctr, ks);
}
if (n == 16)
{
if (encrypt)
{
for (i = 0; i < 16; i++)
x[i] ^= in[i];
for (i = 0; i < 16; i++)
out[i] = in[i] ^ ks[i];
}
else
for (i = 0; i < 16; i++)
{
uint8_t p = (uint8_t)(in[i] ^ ks[i]);
out[i] = p;
x[i] ^= p;
}
}
else if (encrypt)
for (i = 0; i < n; i++)
{
x[pos + i] ^= in[i];
out[i] = in[i] ^ ks[pos + i];
}
else
for (i = 0; i < n; i++)
{
out[i] = in[i] ^ ks[pos + i];
x[pos + i] ^= out[i];
}
off += n;
if (!(off & 15) || off == s->len)
aes_encrypt_block(s->aes, x, x);
in += n;
out += n;
len -= n;
}
s->off = off;
memcpy(s->x, x, 16);
memcpy(s->ctr, ctr, 16);
memcpy(s->ks, ks, 16);
crypto_memzero(x, sizeof(x));
crypto_memzero(ks, sizeof(ks));
}
void aes_ccm_stream_encrypt(struct aes_ccm_stream *s,
const uint8_t *pt, uint8_t *ct, size_t len)
{
aes_ccm_stream_run(s, 1, pt, ct, len);
}
void aes_ccm_stream_decrypt(struct aes_ccm_stream *s,
const uint8_t *ct, uint8_t *pt, size_t len)
{
aes_ccm_stream_run(s, 0, ct, pt, len);
}
void aes_ccm_stream_tag(struct aes_ccm_stream *s, uint8_t *tag)
{
size_t i;
for (i = 0; i < s->tag_len; i++)
tag[i] = s->x[i] ^ s->s0[i];
crypto_memzero(s, sizeof(*s));
}
/* CBC-MAC over B_0 || AAD (length-prefixed, padded) || payload
* (padded), then CTR with A_i counters; the tag is the MAC under
* counter zero. */
static int aes_ccm_core(const struct aes_ctx *ctx, int encrypt,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *in, size_t len, uint8_t *out,
uint8_t *tag, size_t tag_len)
{
uint8_t x[16];
uint8_t ctr[16];
uint8_t s0[16];
size_t L, i, off;
size_t remaining;
if (nonce_len < 7 || nonce_len > 13 || tag_len < 4 || tag_len > 16 || (tag_len & 1))
return -1;
L = 15 - nonce_len;
/* the length must fit L octets; shifted as 64 bits, since a 32-bit
* size_t shifted by 32 (L = 4, the 11-octet nonce SMB uses) is
* undefined and comes back unshifted on MIPS */
if (L < 8 && ((uint64_t)len >> (8 * L)) != 0)
return -1;
/* B_0: flags | nonce | length of the message */
x[0] = (uint8_t)((aad_len ? 0x40 : 0) | (((tag_len - 2) / 2) << 3) | (L - 1));
memcpy(x + 1, nonce, nonce_len);
remaining = len;
for (i = 0; i < L; i++)
{
x[15 - i] = (uint8_t)remaining;
remaining >>= 8;
}
aes_encrypt_block(ctx, x, x);
/* AAD, length-prefixed (2 or 6 octets here; > 2^32 is not needed) */
if (aad_len)
{
uint8_t blk[16];
size_t n;
memset(blk, 0, 16);
if (aad_len < 0xff00)
{
blk[0] = (uint8_t)(aad_len >> 8);
blk[1] = (uint8_t)aad_len;
off = 2;
}
else
{
blk[0] = 0xff; blk[1] = 0xfe;
crypto_store32_be(blk + 2, (uint32_t)aad_len);
off = 6;
}
n = 16 - off;
if (n > aad_len)
n = aad_len;
memcpy(blk + off, aad, n);
for (i = 0; i < 16; i++)
x[i] ^= blk[i];
aes_encrypt_block(ctx, x, x);
aad += n;
aad_len -= n;
while (aad_len)
{
n = aad_len < 16 ? aad_len : 16;
for (i = 0; i < n; i++)
x[i] ^= aad[i];
aes_encrypt_block(ctx, x, x);
aad += n;
aad_len -= n;
}
}
/* Counter blocks: flags = L - 1, nonce, counter */
memset(ctr, 0, 16);
ctr[0] = (uint8_t)(L - 1);
memcpy(ctr + 1, nonce, nonce_len);
aes_encrypt_block(ctx, ctr, s0); /* S_0, for the tag */
off = 0;
while (off < len)
{
uint8_t ks[16];
size_t n = (len - off < 16) ? len - off : 16;
size_t c = off / 16 + 1;
for (i = 0; i < L; i++)
{
ctr[15 - i] = (uint8_t)c;
c >>= 8;
}
aes_encrypt_block(ctx, ctr, ks);
if (encrypt)
{
for (i = 0; i < n; i++)
x[i] ^= in[off + i];
aes_encrypt_block(ctx, x, x);
for (i = 0; i < n; i++)
out[off + i] = in[off + i] ^ ks[i];
}
else
{
for (i = 0; i < n; i++)
{
out[off + i] = in[off + i] ^ ks[i];
x[i] ^= out[off + i];
}
aes_encrypt_block(ctx, x, x);
}
off += n;
}
for (i = 0; i < tag_len; i++)
tag[i] = x[i] ^ s0[i];
crypto_memzero(x, sizeof(x));
crypto_memzero(s0, sizeof(s0));
return 0;
}
int aes_ccm_encrypt(const struct aes_ctx *ctx,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *pt, size_t pt_len,
uint8_t *ct, uint8_t *tag, size_t tag_len)
{
return aes_ccm_core(ctx, 1, nonce, nonce_len, aad, aad_len,
pt, pt_len, ct, tag, tag_len);
}
int aes_ccm_decrypt(const struct aes_ctx *ctx,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *ct, size_t ct_len,
const uint8_t *tag, size_t tag_len, uint8_t *pt)
{
uint8_t calc[16];
if (aes_ccm_core(ctx, 0, nonce, nonce_len, aad, aad_len,
ct, ct_len, pt, calc, tag_len) != 0)
return -1;
if (!crypto_memeq_ct(calc, tag, tag_len))
{
crypto_memzero(pt, ct_len);
return -1;
}
return 0;
}
/* GHASH, scalar: 4-bit tables of H (16 entries, built at init) and a
* 16-entry reduction table, two lookups per octet. */
static const uint64_t aes_gcm_last4[16] = {
0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0,
0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0
};
static void aes_gcm_mul(const struct aes_gcm_ctx *ctx,
uint64_t *y_hi, uint64_t *y_lo)
{
uint8_t x[16];
uint64_t zh, zl, rem;
int i;
unsigned lo, hi;
crypto_store64_be(x, *y_hi);
crypto_store64_be(x + 8, *y_lo);
lo = x[15] & 0x0f;
zh = ctx->hh[lo];
zl = ctx->hl[lo];
for (i = 15; i >= 0; i--)
{
lo = x[i] & 0x0f;
hi = (x[i] >> 4) & 0x0f;
if (i != 15)
{
rem = zl & 0x0f;
zl = (zh << 60) | (zl >> 4);
zh = (zh >> 4);
zh ^= aes_gcm_last4[rem] << 48;
zh ^= ctx->hh[lo];
zl ^= ctx->hl[lo];
}
rem = zl & 0x0f;
zl = (zh << 60) | (zl >> 4);
zh = (zh >> 4);
zh ^= aes_gcm_last4[rem] << 48;
zh ^= ctx->hh[hi];
zl ^= ctx->hl[hi];
}
*y_hi = zh;
*y_lo = zl;
}
static void aes_gcm_ghash_update(const struct aes_gcm_ctx *ctx,
uint64_t *y_hi, uint64_t *y_lo, const uint8_t *data, size_t len)
{
#if defined(AES_HAVE_X86_PATH)
if (aes_x86_available())
{
aes_x86_ghash(ctx, y_hi, y_lo, data, len);
return;
}
#elif defined(AES_HAVE_ARM_PATH)
if (aes_arm_available())
{
aes_arm_ghash(ctx, y_hi, y_lo, data, len);
return;
}
#endif
while (len)
{
uint8_t blk[16];
size_t n = (len < 16) ? len : 16;
if (n < 16)
{
memset(blk, 0, 16);
memcpy(blk, data, n);
data = blk;
}
*y_hi ^= crypto_load64_be(data);
*y_lo ^= crypto_load64_be(data + 8);
aes_gcm_mul(ctx, y_hi, y_lo);
data += n;
len -= n;
}
}
static void aes_gcm_tables(struct aes_gcm_ctx *ctx)
{
uint64_t vh = ctx->h_hi, vl = ctx->h_lo;
int i, j;
ctx->hh[0] = 0; ctx->hl[0] = 0;
ctx->hh[8] = vh; ctx->hl[8] = vl;
for (i = 4; i > 0; i >>= 1)
{
uint32_t t = (uint32_t)(vl & 1) * 0xe1000000u;
vl = (vh << 63) | (vl >> 1);
vh = (vh >> 1) ^ ((uint64_t)t << 32);
ctx->hl[i] = vl;
ctx->hh[i] = vh;
}
for (i = 2; i <= 8; i *= 2)
{
vh = ctx->hh[i];
vl = ctx->hl[i];
for (j = 1; j < i; j++)
{
ctx->hh[i + j] = vh ^ ctx->hh[j];
ctx->hl[i + j] = vl ^ ctx->hl[j];
}
}
}
int aes_gcm_init(struct aes_gcm_ctx *ctx,
const uint8_t *key, size_t key_len)
{
uint8_t h[16];
if (aes_init(&ctx->aes, key, key_len) != 0)
return -1;
memset(h, 0, 16);
aes_encrypt_block(&ctx->aes, h, h);
ctx->h_hi = crypto_load64_be(h);
ctx->h_lo = crypto_load64_be(h + 8);
aes_gcm_tables(ctx);
crypto_memzero(h, sizeof(h));
return 0;
}
static void aes_gcm_j0(const struct aes_gcm_ctx *ctx,
const uint8_t *nonce, size_t nonce_len, uint8_t *j0)
{
if (nonce_len == 12)
{
memcpy(j0, nonce, 12);
j0[12] = 0; j0[13] = 0; j0[14] = 0; j0[15] = 1;
}
else
{
uint64_t y_hi = 0, y_lo = 0;
aes_gcm_ghash_update(ctx, &y_hi, &y_lo, nonce, nonce_len);
y_lo ^= (uint64_t)nonce_len * 8;
aes_gcm_mul(ctx, &y_hi, &y_lo);
crypto_store64_be(j0, y_hi);
crypto_store64_be(j0 + 8, y_lo);
}
}
static void aes_gcm_tag(const struct aes_gcm_ctx *ctx, const uint8_t *j0,
const uint8_t *aad, size_t aad_len,
const uint8_t *ct, size_t ct_len, uint8_t *tag)
{
unsigned i;
uint64_t y_hi = 0, y_lo = 0;
uint8_t ek[16];
aes_gcm_ghash_update(ctx, &y_hi, &y_lo, aad, aad_len);
aes_gcm_ghash_update(ctx, &y_hi, &y_lo, ct, ct_len);
y_hi ^= (uint64_t)aad_len * 8;
y_lo ^= (uint64_t)ct_len * 8;
aes_gcm_mul(ctx, &y_hi, &y_lo);
aes_encrypt_block(&ctx->aes, j0, ek);
crypto_store64_be(tag, y_hi);
crypto_store64_be(tag + 8, y_lo);
for (i = 0; i < 16; i++)
tag[i] ^= ek[i];
crypto_memzero(ek, sizeof(ek));
}
int aes_gcm_encrypt(const struct aes_gcm_ctx *ctx,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *pt, size_t pt_len,
uint8_t *ct, uint8_t *tag)
{
uint8_t j0[16];
uint8_t ctr[16];
aes_gcm_j0(ctx, nonce, nonce_len, j0);
memcpy(ctr, j0, 16);
aes_ctr_inc32(ctr);
aes_ctr_crypt(&ctx->aes, ctr, pt, ct, pt_len);
aes_gcm_tag(ctx, j0, aad, aad_len, ct, pt_len, tag);
return 0;
}
int aes_gcm_decrypt(const struct aes_gcm_ctx *ctx,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *ct, size_t ct_len,
const uint8_t *tag, uint8_t *pt)
{
uint8_t j0[16];
uint8_t ctr[16];
uint8_t calc[16];
int ok;
aes_gcm_j0(ctx, nonce, nonce_len, j0);
aes_gcm_tag(ctx, j0, aad, aad_len, ct, ct_len, calc);
ok = crypto_memeq_ct(calc, tag, 16);
crypto_memzero(calc, sizeof(calc));
if (!ok)
return -1;
memcpy(ctr, j0, 16);
aes_ctr_inc32(ctr);
aes_ctr_crypt(&ctx->aes, ctr, ct, pt, ct_len);
return 0;
}
void aes_gcm_stream_init(struct aes_gcm_stream *s,
const struct aes_gcm_ctx *ctx, const uint8_t *nonce, size_t nonce_len)
{
s->g = ctx;
s->y_hi = 0;
s->y_lo = 0;
s->aad_len = 0;
s->ct_len = 0;
s->part_len = 0;
aes_gcm_j0(ctx, nonce, nonce_len, s->j0);
}
/* GHASH takes whole blocks; a piece's tail waits in part for the
* next piece of the same input, and is padded where that input ends. */
static void aes_gcm_stream_feed(struct aes_gcm_stream *s,
const uint8_t *data, size_t len)
{
size_t bulk;
if (s->part_len)
{
size_t take = 16 - s->part_len;
if (take > len)
take = len;
memcpy(s->part + s->part_len, data, take);
s->part_len += take;
data += take;
len -= take;
if (s->part_len < 16)
return;
aes_gcm_ghash_update(s->g, &s->y_hi, &s->y_lo, s->part, 16);
s->part_len = 0;
}
bulk = len & ~(size_t)15;
if (bulk)
aes_gcm_ghash_update(s->g, &s->y_hi, &s->y_lo, data, bulk);
memcpy(s->part, data + bulk, len - bulk);
s->part_len = len - bulk;
}
static void aes_gcm_stream_pad(struct aes_gcm_stream *s)
{
if (s->part_len)
{
aes_gcm_ghash_update(s->g, &s->y_hi, &s->y_lo, s->part, s->part_len);
s->part_len = 0;
}
}
void aes_gcm_stream_aad(struct aes_gcm_stream *s,
const uint8_t *aad, size_t len)
{
aes_gcm_stream_feed(s, aad, len);
s->aad_len += len;
}
void aes_gcm_stream_hash_ct(struct aes_gcm_stream *s,
const uint8_t *ct, size_t len)
{
if (!s->ct_len)
aes_gcm_stream_pad(s); /* the AAD ends here */
aes_gcm_stream_feed(s, ct, len);
s->ct_len += len;
}
void aes_gcm_stream_tag(struct aes_gcm_stream *s, uint8_t *tag)
{
unsigned i;
uint8_t ek[16];
aes_gcm_stream_pad(s);
s->y_hi ^= s->aad_len * 8;
s->y_lo ^= s->ct_len * 8;
aes_gcm_mul(s->g, &s->y_hi, &s->y_lo);
aes_encrypt_block(&s->g->aes, s->j0, ek);
crypto_store64_be(tag, s->y_hi);
crypto_store64_be(tag + 8, s->y_lo);
for (i = 0; i < 16; i++)
tag[i] ^= ek[i];
crypto_memzero(ek, sizeof(ek));
}
void aes_gcm_stream_crypt(const struct aes_gcm_stream *s, uint64_t off,
const uint8_t *in, uint8_t *out, size_t len)
{
uint8_t ctr[16];
size_t pos = (size_t)(off & 15);
/* the ciphertext's block b runs under counter J0 + 1 + b */
memcpy(ctr, s->j0, 16);
crypto_store32_be(ctr + 12, crypto_load32_be(s->j0 + 12)
+ 1 + (uint32_t)(off >> 4));
if (pos)
{
uint8_t ks[16];
size_t i, n = 16 - pos;
if (n > len)
n = len;
aes_encrypt_block(&s->g->aes, ctr, ks);
aes_ctr_inc32(ctr);
for (i = 0; i < n; i++)
out[i] = in[i] ^ ks[pos + i];
crypto_memzero(ks, sizeof(ks));
in += n;
out += n;
len -= n;
}
if (len)
aes_ctr_crypt(&s->g->aes, ctr, in, out, len);
crypto_memzero(ctr, sizeof(ctr));
}
#define QR(a, b, c, d) \
a += b; d ^= a; d = crypto_rotl32(d, 16); \
c += d; b ^= c; b = crypto_rotl32(b, 12); \
a += b; d ^= a; d = crypto_rotl32(d, 8); \
c += d; b ^= c; b = crypto_rotl32(b, 7)
static void chacha20_core(const uint8_t *key, const uint8_t *nonce,
uint32_t counter, uint32_t *out)
{
unsigned i;
uint32_t s[16];
uint32_t x[16];
s[0] = 0x61707865; s[1] = 0x3320646e;
s[2] = 0x79622d32; s[3] = 0x6b206574;
for (i = 0; i < 8; i++)
s[4 + i] = crypto_load32_le(key + 4 * i);
s[12] = counter;
s[13] = crypto_load32_le(nonce);
s[14] = crypto_load32_le(nonce + 4);
s[15] = crypto_load32_le(nonce + 8);
memcpy(x, s, sizeof(x));
for (i = 0; i < 10; i++)
{
QR(x[0], x[4], x[ 8], x[12]);
QR(x[1], x[5], x[ 9], x[13]);
QR(x[2], x[6], x[10], x[14]);
QR(x[3], x[7], x[11], x[15]);
QR(x[0], x[5], x[10], x[15]);
QR(x[1], x[6], x[11], x[12]);
QR(x[2], x[7], x[ 8], x[13]);
QR(x[3], x[4], x[ 9], x[14]);
}
for (i = 0; i < 16; i++)
out[i] = x[i] + s[i];
}
void chacha20_block(const uint8_t *key, const uint8_t *nonce,
uint32_t counter, uint8_t *out)
{
unsigned i;
uint32_t w[16];
chacha20_core(key, nonce, counter, w);
for (i = 0; i < 16; i++)
crypto_store32_le(out + 4 * i, w[i]);
}
#if defined(AES_HAVE_X86_PATH)
static retro_atomic_int_t chacha_x86_ok;
static int chacha_x86_available(void)
{
int v = retro_atomic_load_acquire_int(&chacha_x86_ok);
if (!v)
{
v = (cpu_features_get() & RETRO_SIMD_SSSE3) ? 2 : 1;
retro_atomic_store_release_int(&chacha_x86_ok, v);
}
return v == 2;
}
#define CHACHA_ROTL(v, n) _mm_or_si128(_mm_slli_epi32(v, n), _mm_srli_epi32(v, 32 - (n)))
#define CHACHA_QR4(a, b, c, d) \
a = _mm_add_epi32(a, b); d = _mm_shuffle_epi8(_mm_xor_si128(d, a), r16); \
c = _mm_add_epi32(c, d); b = CHACHA_ROTL(_mm_xor_si128(b, c), 12); \
a = _mm_add_epi32(a, b); d = _mm_shuffle_epi8(_mm_xor_si128(d, a), r8); \
c = _mm_add_epi32(c, d); b = CHACHA_ROTL(_mm_xor_si128(b, c), 7)
/* Four blocks at once, one state word per vector and one block per
* lane; the 16- and 8-bit rotations are byte shuffles. @in to @out,
* whole groups of 256 octets; returns the octets done. */
__attribute__((target("ssse3")))
static size_t chacha20_x86_xor(const uint32_t *s, uint32_t counter,
const uint8_t *in, uint8_t *out, size_t len)
{
const __m128i r16 = _mm_set_epi8(13,12,15,14, 9,8,11,10, 5,4,7,6, 1,0,3,2);
const __m128i r8 = _mm_set_epi8(14,13,12,15, 10,9,8,11, 6,5,4,7, 2,1,0,3);
size_t done = 0;
while (len - done >= 256)
{
__m128i x[16], o[16];
unsigned i, g;
for (i = 0; i < 16; i++)
o[i] = _mm_set1_epi32((int)s[i]);
o[12] = _mm_add_epi32(_mm_set1_epi32((int)counter), _mm_set_epi32(3, 2, 1, 0));
for (i = 0; i < 16; i++)
x[i] = o[i];
for (i = 0; i < 10; i++)
{
CHACHA_QR4(x[0], x[4], x[ 8], x[12]);
CHACHA_QR4(x[1], x[5], x[ 9], x[13]);
CHACHA_QR4(x[2], x[6], x[10], x[14]);
CHACHA_QR4(x[3], x[7], x[11], x[15]);
CHACHA_QR4(x[0], x[5], x[10], x[15]);
CHACHA_QR4(x[1], x[6], x[11], x[12]);
CHACHA_QR4(x[2], x[7], x[ 8], x[13]);
CHACHA_QR4(x[3], x[4], x[ 9], x[14]);
}
for (i = 0; i < 16; i++)
x[i] = _mm_add_epi32(x[i], o[i]);
/* four words of each of the four blocks per group: transposed
* back into block order and XORed in place */
for (g = 0; g < 4; g++)
{
__m128i t0 = _mm_unpacklo_epi32(x[4 * g], x[4 * g + 1]);
__m128i t1 = _mm_unpacklo_epi32(x[4 * g + 2], x[4 * g + 3]);
__m128i t2 = _mm_unpackhi_epi32(x[4 * g], x[4 * g + 1]);
__m128i t3 = _mm_unpackhi_epi32(x[4 * g + 2], x[4 * g + 3]);
const uint8_t *ip = in + done + 16 * g;
uint8_t *op = out + done + 16 * g;
_mm_storeu_si128((__m128i*)(op), _mm_xor_si128(_mm_unpacklo_epi64(t0, t1), _mm_loadu_si128((const __m128i*)(ip))));
_mm_storeu_si128((__m128i*)(op + 64), _mm_xor_si128(_mm_unpackhi_epi64(t0, t1), _mm_loadu_si128((const __m128i*)(ip + 64))));
_mm_storeu_si128((__m128i*)(op + 128), _mm_xor_si128(_mm_unpacklo_epi64(t2, t3), _mm_loadu_si128((const __m128i*)(ip + 128))));
_mm_storeu_si128((__m128i*)(op + 192), _mm_xor_si128(_mm_unpackhi_epi64(t2, t3), _mm_loadu_si128((const __m128i*)(ip + 192))));
}
counter += 4;
done += 256;
}
return done;
}
#undef CHACHA_QR4
#undef CHACHA_ROTL
#endif
/* every AArch64 core, and ARMv7 built for NEON (the Vita, 32-bit
* Android); older GCC spells the macro __ARM_NEON__ */
#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && !defined(_MSC_VER) && !defined(CHACHA_NO_NEON)
#include <arm_neon.h>
#define CHACHA_HAVE_NEON 1
/* rotl by 16 is a halfword swap; the others a shift and a shift-insert */
#define CHACHA_NROT(v, n) vsriq_n_u32(vshlq_n_u32(v, n), v, 32 - (n))
#define CHACHA_NQR(a, b, c, d) \
a = vaddq_u32(a, b); d = vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(veorq_u32(d, a)))); \
c = vaddq_u32(c, d); b = CHACHA_NROT(veorq_u32(b, c), 12); \
a = vaddq_u32(a, b); d = CHACHA_NROT(veorq_u32(d, a), 8); \
c = vaddq_u32(c, d); b = CHACHA_NROT(veorq_u32(b, c), 7)
/* Four blocks at once on NEON: one state word per vector and one block
* per lane, as on SSSE3. Whole groups of 256 octets; returns the octets
* done. */
static size_t chacha20_neon_xor(const uint32_t *s, uint32_t counter,
const uint8_t *in, uint8_t *out, size_t len)
{
static const uint32_t lane[4] = { 0, 1, 2, 3 };
size_t done = 0;
while (len - done >= 256)
{
uint32x4_t x[16], o[16];
unsigned i, g;
for (i = 0; i < 16; i++)
o[i] = vdupq_n_u32(s[i]);
o[12] = vaddq_u32(vdupq_n_u32(counter), vld1q_u32(lane));
for (i = 0; i < 16; i++)
x[i] = o[i];
for (i = 0; i < 10; i++)
{
CHACHA_NQR(x[0], x[4], x[ 8], x[12]);
CHACHA_NQR(x[1], x[5], x[ 9], x[13]);
CHACHA_NQR(x[2], x[6], x[10], x[14]);
CHACHA_NQR(x[3], x[7], x[11], x[15]);
CHACHA_NQR(x[0], x[5], x[10], x[15]);
CHACHA_NQR(x[1], x[6], x[11], x[12]);
CHACHA_NQR(x[2], x[7], x[ 8], x[13]);
CHACHA_NQR(x[3], x[4], x[ 9], x[14]);
}
for (i = 0; i < 16; i++)
x[i] = vaddq_u32(x[i], o[i]);
/* each group of four words back into block order: a 4x4
* transpose, then XORed in */
for (g = 0; g < 4; g++)
{
uint32x4x2_t p = vtrnq_u32(x[4 * g], x[4 * g + 1]);
uint32x4x2_t q = vtrnq_u32(x[4 * g + 2], x[4 * g + 3]);
uint32x4_t r0 = vcombine_u32(vget_low_u32(p.val[0]), vget_low_u32(q.val[0]));
uint32x4_t r1 = vcombine_u32(vget_low_u32(p.val[1]), vget_low_u32(q.val[1]));
uint32x4_t r2 = vcombine_u32(vget_high_u32(p.val[0]), vget_high_u32(q.val[0]));
uint32x4_t r3 = vcombine_u32(vget_high_u32(p.val[1]), vget_high_u32(q.val[1]));
const uint8_t *ip = in + done + 16 * g;
uint8_t *op = out + done + 16 * g;
vst1q_u8(op, veorq_u8(vreinterpretq_u8_u32(r0), vld1q_u8(ip)));
vst1q_u8(op + 64, veorq_u8(vreinterpretq_u8_u32(r1), vld1q_u8(ip + 64)));
vst1q_u8(op + 128, veorq_u8(vreinterpretq_u8_u32(r2), vld1q_u8(ip + 128)));
vst1q_u8(op + 192, veorq_u8(vreinterpretq_u8_u32(r3), vld1q_u8(ip + 192)));
}
counter += 4;
done += 256;
}
return done;
}
#undef CHACHA_NQR
#undef CHACHA_NROT
#endif
void chacha20_xor(const uint8_t *key, const uint8_t *nonce,
uint32_t counter, const uint8_t *in, uint8_t *out, size_t len)
{
uint32_t w[16];
unsigned i;
#if defined(AES_HAVE_X86_PATH) || defined(CHACHA_HAVE_NEON)
#if defined(AES_HAVE_X86_PATH)
if (len >= 256 && chacha_x86_available())
#else
if (len >= 256)
#endif
{
uint32_t s[16];
size_t done;
s[0] = 0x61707865; s[1] = 0x3320646e;
s[2] = 0x79622d32; s[3] = 0x6b206574;
for (i = 0; i < 8; i++)
s[4 + i] = crypto_load32_le(key + 4 * i);
s[12] = counter;
s[13] = crypto_load32_le(nonce);
s[14] = crypto_load32_le(nonce + 4);
s[15] = crypto_load32_le(nonce + 8);
#if defined(AES_HAVE_X86_PATH)
done = chacha20_x86_xor(s, counter, in, out, len);
#else
done = chacha20_neon_xor(s, counter, in, out, len);
#endif
counter += (uint32_t)(done / 64);
in += done;
out += done;
len -= done;
crypto_memzero(s, sizeof(s));
}
#endif
/* Whole blocks a word at a time, no keystream bounce buffer. */
while (len >= 64)
{
chacha20_core(key, nonce, counter, w);
counter++;
for (i = 0; i < 16; i++)
crypto_store32_le(out + 4 * i, crypto_load32_le(in + 4 * i) ^ w[i]);
in += 64;
out += 64;
len -= 64;
}
if (len)
{
uint8_t ks[64];
chacha20_block(key, nonce, counter, ks);
for (i = 0; i < len; i++)
out[i] = in[i] ^ ks[i];
crypto_memzero(ks, sizeof(ks));
}
crypto_memzero(w, sizeof(w));
}
#define P1305_MASK 0x3ffffff
void poly1305_init(struct poly1305_ctx *ctx, const uint8_t *key)
{
uint32_t t0 = crypto_load32_le(key);
uint32_t t1 = crypto_load32_le(key + 4);
uint32_t t2 = crypto_load32_le(key + 8);
uint32_t t3 = crypto_load32_le(key + 12);
/* r &= 0xffffffc0ffffffc0ffffffc0fffffff, split into 26-bit limbs. */
ctx->r[0] = t0 & 0x3ffffff; t0 >>= 26; t0 |= t1 << 6;
ctx->r[1] = t0 & 0x3ffff03; t1 >>= 20; t1 |= t2 << 12;
ctx->r[2] = t1 & 0x3ffc0ff; t2 >>= 14; t2 |= t3 << 18;
ctx->r[3] = t2 & 0x3f03fff; t3 >>= 8;
ctx->r[4] = t3 & 0x00fffff;
ctx->h[0] = ctx->h[1] = ctx->h[2] = ctx->h[3] = ctx->h[4] = 0;
ctx->pad[0] = crypto_load32_le(key + 16);
ctx->pad[1] = crypto_load32_le(key + 20);
ctx->pad[2] = crypto_load32_le(key + 24);
ctx->pad[3] = crypto_load32_le(key + 28);
ctx->leftover = 0;
ctx->final = 0;
}
#if defined(__SIZEOF_INT128__) && !defined(POLY1305_NO_128)
/* 64-bit machines: the same arithmetic in 44-bit limbs, 9 multiplies
* per block instead of 25. The context keeps 26-bit limbs; they are
* turned into 44-bit ones on the way in and back on the way out, which
* costs a few operations a call against every block's savings. */
__extension__ typedef unsigned __int128 p1305_u128;
#define P1305_M44 ((uint64_t)0xfffffffffffULL)
#define P1305_M42 ((uint64_t)0x3ffffffffffULL)
static void poly1305_blocks_44(struct poly1305_ctx *ctx,
const uint8_t *m, size_t len)
{
const uint64_t hibit = ctx->final ? 0 : ((uint64_t)1 << 40);
uint64_t r0, r1, r2, s1, s2, h0, h1, h2, lo;
uint32_t c, g[5];
p1305_u128 up;
unsigned i;
/* r: 124 bits, its 26-bit limbs exact */
lo = (uint64_t)ctx->r[0] | ((uint64_t)ctx->r[1] << 26) | ((uint64_t)ctx->r[2] << 52);
up = (p1305_u128)(ctx->r[2] >> 12) | ((p1305_u128)ctx->r[3] << 14)
| ((p1305_u128)ctx->r[4] << 40);
r0 = lo & P1305_M44;
r1 = ((lo >> 44) | (uint64_t)(up << 20)) & P1305_M44;
r2 = (uint64_t)(up >> 24) & P1305_M42;
s1 = r1 * (5 << 2);
s2 = r2 * (5 << 2);
/* h: partly reduced 26-bit limbs, carried first so each fits */
for (i = 0; i < 5; i++)
g[i] = ctx->h[i];
c = g[0] >> 26; g[0] &= P1305_MASK; g[1] += c;
c = g[1] >> 26; g[1] &= P1305_MASK; g[2] += c;
c = g[2] >> 26; g[2] &= P1305_MASK; g[3] += c;
c = g[3] >> 26; g[3] &= P1305_MASK; g[4] += c;
lo = (uint64_t)g[0] | ((uint64_t)g[1] << 26) | ((uint64_t)(g[2] & 0xfff) << 52);
up = (p1305_u128)(g[2] >> 12) + ((p1305_u128)g[3] << 14) + ((p1305_u128)g[4] << 40);
h0 = lo & P1305_M44;
h1 = ((lo >> 44) | (uint64_t)(up << 20)) & P1305_M44;
h2 = (uint64_t)(up >> 24);
while (len >= 16)
{
uint64_t t0 = (uint64_t)crypto_load32_le(m) | ((uint64_t)crypto_load32_le(m + 4) << 32);
uint64_t t1 = (uint64_t)crypto_load32_le(m + 8) | ((uint64_t)crypto_load32_le(m + 12) << 32);
p1305_u128 d0, d1, d2;
uint64_t cc;
h0 += t0 & P1305_M44;
h1 += ((t0 >> 44) | (t1 << 20)) & P1305_M44;
h2 += ((t1 >> 24) & P1305_M42) | hibit;
d0 = (p1305_u128)h0 * r0 + (p1305_u128)h1 * s2 + (p1305_u128)h2 * s1;
d1 = (p1305_u128)h0 * r1 + (p1305_u128)h1 * r0 + (p1305_u128)h2 * s2;
d2 = (p1305_u128)h0 * r2 + (p1305_u128)h1 * r1 + (p1305_u128)h2 * r0;
cc = (uint64_t)(d0 >> 44); h0 = (uint64_t)d0 & P1305_M44;
d1 += cc; cc = (uint64_t)(d1 >> 44); h1 = (uint64_t)d1 & P1305_M44;
d2 += cc; cc = (uint64_t)(d2 >> 42); h2 = (uint64_t)d2 & P1305_M42;
h0 += cc * 5; cc = h0 >> 44; h0 &= P1305_M44;
h1 += cc;
m += 16;
len -= 16;
}
/* back to 26-bit limbs: h0 | h1 << 44 | h2 << 88 */
ctx->h[0] = (uint32_t)(h0 & P1305_MASK);
ctx->h[1] = (uint32_t)(((h0 >> 26) | (h1 << 18)) & P1305_MASK);
ctx->h[2] = (uint32_t)((h1 >> 8) & P1305_MASK);
ctx->h[3] = (uint32_t)(((h1 >> 34) | (h2 << 10)) & P1305_MASK);
ctx->h[4] = (uint32_t)(h2 >> 16);
}
#endif
static void poly1305_blocks(struct poly1305_ctx *ctx,
const uint8_t *m, size_t len)
{
#if defined(__SIZEOF_INT128__) && !defined(POLY1305_NO_128)
poly1305_blocks_44(ctx, m, len);
}
#else
const uint32_t hibit = ctx->final ? 0 : ((uint32_t)1 << 24);
uint32_t r0 = ctx->r[0], r1 = ctx->r[1], r2 = ctx->r[2];
uint32_t r3 = ctx->r[3], r4 = ctx->r[4];
uint32_t s1 = r1 * 5, s2 = r2 * 5, s3 = r3 * 5, s4 = r4 * 5;
uint32_t h0 = ctx->h[0], h1 = ctx->h[1], h2 = ctx->h[2];
uint32_t h3 = ctx->h[3], h4 = ctx->h[4];
while (len >= 16)
{
uint64_t d0, d1, d2, d3, d4;
uint32_t c;
uint32_t t0 = crypto_load32_le(m);
uint32_t t1 = crypto_load32_le(m + 4);
uint32_t t2 = crypto_load32_le(m + 8);
uint32_t t3 = crypto_load32_le(m + 12);
/* h += m[i] */
h0 += t0 & P1305_MASK;
h1 += ((t0 >> 26) | (t1 << 6)) & P1305_MASK;
h2 += ((t1 >> 20) | (t2 << 12)) & P1305_MASK;
h3 += ((t2 >> 14) | (t3 << 18)) & P1305_MASK;
h4 += (t3 >> 8) | hibit;
/* h *= r, mod 2^130 - 5 */
d0 = (uint64_t)h0 * r0 + (uint64_t)h1 * s4 + (uint64_t)h2 * s3
+ (uint64_t)h3 * s2 + (uint64_t)h4 * s1;
d1 = (uint64_t)h0 * r1 + (uint64_t)h1 * r0 + (uint64_t)h2 * s4
+ (uint64_t)h3 * s3 + (uint64_t)h4 * s2;
d2 = (uint64_t)h0 * r2 + (uint64_t)h1 * r1 + (uint64_t)h2 * r0
+ (uint64_t)h3 * s4 + (uint64_t)h4 * s3;
d3 = (uint64_t)h0 * r3 + (uint64_t)h1 * r2 + (uint64_t)h2 * r1
+ (uint64_t)h3 * r0 + (uint64_t)h4 * s4;
d4 = (uint64_t)h0 * r4 + (uint64_t)h1 * r3 + (uint64_t)h2 * r2
+ (uint64_t)h3 * r1 + (uint64_t)h4 * r0;
/* partial reduction */
c = (uint32_t)(d0 >> 26); h0 = (uint32_t)d0 & P1305_MASK;
d1 += c; c = (uint32_t)(d1 >> 26); h1 = (uint32_t)d1 & P1305_MASK;
d2 += c; c = (uint32_t)(d2 >> 26); h2 = (uint32_t)d2 & P1305_MASK;
d3 += c; c = (uint32_t)(d3 >> 26); h3 = (uint32_t)d3 & P1305_MASK;
d4 += c; c = (uint32_t)(d4 >> 26); h4 = (uint32_t)d4 & P1305_MASK;
h0 += c * 5; c = h0 >> 26; h0 &= P1305_MASK;
h1 += c;
m += 16;
len -= 16;
}
ctx->h[0] = h0; ctx->h[1] = h1; ctx->h[2] = h2;
ctx->h[3] = h3; ctx->h[4] = h4;
}
#endif
void poly1305_update(struct poly1305_ctx *ctx,
const uint8_t *m, size_t len)
{
size_t want;
if (ctx->leftover)
{
want = 16 - ctx->leftover;
if (want > len)
want = len;
memcpy(ctx->buffer + ctx->leftover, m, want);
len -= want;
m += want;
ctx->leftover += want;
if (ctx->leftover < 16)
return;
poly1305_blocks(ctx, ctx->buffer, 16);
ctx->leftover = 0;
}
if (len >= 16)
{
want = len & ~(size_t)15;
poly1305_blocks(ctx, m, want);
m += want;
len -= want;
}
if (len)
{
memcpy(ctx->buffer, m, len);
ctx->leftover = len;
}
}
void poly1305_final(struct poly1305_ctx *ctx, uint8_t *tag)
{
uint32_t h0, h1, h2, h3, h4, c;
uint32_t g0, g1, g2, g3, g4;
uint64_t f;
uint32_t mask;
if (ctx->leftover)
{
size_t i = ctx->leftover;
ctx->buffer[i++] = 1;
for (; i < 16; i++)
ctx->buffer[i] = 0;
ctx->final = 1;
poly1305_blocks(ctx, ctx->buffer, 16);
}
h0 = ctx->h[0]; h1 = ctx->h[1]; h2 = ctx->h[2];
h3 = ctx->h[3]; h4 = ctx->h[4];
/* full carry */
c = h1 >> 26; h1 &= P1305_MASK;
h2 += c; c = h2 >> 26; h2 &= P1305_MASK;
h3 += c; c = h3 >> 26; h3 &= P1305_MASK;
h4 += c; c = h4 >> 26; h4 &= P1305_MASK;
h0 += c * 5; c = h0 >> 26; h0 &= P1305_MASK;
h1 += c;
/* g = h + 5 - 2^130; select g when it did not go negative */
g0 = h0 + 5; c = g0 >> 26; g0 &= P1305_MASK;
g1 = h1 + c; c = g1 >> 26; g1 &= P1305_MASK;
g2 = h2 + c; c = g2 >> 26; g2 &= P1305_MASK;
g3 = h3 + c; c = g3 >> 26; g3 &= P1305_MASK;
g4 = h4 + c - ((uint32_t)1 << 26);
mask = (g4 >> 31) - 1;
g0 &= mask; g1 &= mask; g2 &= mask; g3 &= mask; g4 &= mask;
mask = ~mask;
h0 = (h0 & mask) | g0;
h1 = (h1 & mask) | g1;
h2 = (h2 & mask) | g2;
h3 = (h3 & mask) | g3;
h4 = (h4 & mask) | g4;
/* h = h % 2^128 as four 32-bit words */
h0 = ((h0 ) | (h1 << 26));
h1 = ((h1 >> 6) | (h2 << 20));
h2 = ((h2 >> 12) | (h3 << 14));
h3 = ((h3 >> 18) | (h4 << 8));
/* tag = (h + pad) % 2^128 */
f = (uint64_t)h0 + ctx->pad[0]; h0 = (uint32_t)f;
f = (uint64_t)h1 + ctx->pad[1] + (f >> 32); h1 = (uint32_t)f;
f = (uint64_t)h2 + ctx->pad[2] + (f >> 32); h2 = (uint32_t)f;
f = (uint64_t)h3 + ctx->pad[3] + (f >> 32); h3 = (uint32_t)f;
crypto_store32_le(tag, h0);
crypto_store32_le(tag + 4, h1);
crypto_store32_le(tag + 8, h2);
crypto_store32_le(tag + 12, h3);
crypto_memzero(ctx, sizeof(*ctx));
}
void poly1305_auth(uint8_t *tag, const uint8_t *m, size_t len,
const uint8_t *key)
{
struct poly1305_ctx ctx;
poly1305_init(&ctx, key);
poly1305_update(&ctx, m, len);
poly1305_final(&ctx, tag);
}
static const uint8_t aead_zero[16] = { 0 };
/* Poly1305 over pad16(aad) || pad16(ct) || le64(aad_len) || le64(ct_len),
* keyed with ChaCha20 block 0. */
static void aead_chacha_tag(const uint8_t *key, const uint8_t *nonce,
const uint8_t *aad, size_t aad_len,
const uint8_t *ct, size_t ct_len, uint8_t *tag)
{
struct poly1305_ctx p;
uint8_t otk[64];
uint8_t lens[16];
chacha20_block(key, nonce, 0, otk);
poly1305_init(&p, otk);
poly1305_update(&p, aad, aad_len);
if (aad_len & 15)
poly1305_update(&p, aead_zero, 16 - (aad_len & 15));
poly1305_update(&p, ct, ct_len);
if (ct_len & 15)
poly1305_update(&p, aead_zero, 16 - (ct_len & 15));
crypto_store64_le(lens, (uint64_t)aad_len);
crypto_store64_le(lens + 8, (uint64_t)ct_len);
poly1305_update(&p, lens, 16);
poly1305_final(&p, tag);
crypto_memzero(otk, sizeof(otk));
}
int aead_encrypt(enum aead_alg alg,
const uint8_t *key, size_t key_len,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *pt, size_t pt_len,
uint8_t *ct, uint8_t *tag, size_t tag_len)
{
if (key_len != AEAD_KEY_SIZE || nonce_len != AEAD_NONCE_SIZE
|| tag_len != AEAD_TAG_SIZE)
return -1;
switch (alg)
{
case AEAD_CHACHA20_POLY1305:
chacha20_xor(key, nonce, 1, pt, ct, pt_len);
aead_chacha_tag(key, nonce, aad, aad_len, ct, pt_len, tag);
return 0;
case AEAD_AES256_GCM:
{
struct aes_gcm_ctx g;
int ret = aes_gcm_init(&g, key, key_len);
if (ret == 0)
ret = aes_gcm_encrypt(&g, nonce, nonce_len, aad, aad_len,
pt, pt_len, ct, tag);
crypto_memzero(&g, sizeof(g));
return ret;
}
default:
break;
}
return -1;
}
int aead_decrypt(enum aead_alg alg,
const uint8_t *key, size_t key_len,
const uint8_t *nonce, size_t nonce_len,
const uint8_t *aad, size_t aad_len,
const uint8_t *ct, size_t ct_len,
const uint8_t *tag, size_t tag_len,
uint8_t *pt)
{
if (key_len != AEAD_KEY_SIZE || nonce_len != AEAD_NONCE_SIZE
|| tag_len != AEAD_TAG_SIZE)
return -1;
switch (alg)
{
case AEAD_CHACHA20_POLY1305:
{
uint8_t calc[16];
int ok;
aead_chacha_tag(key, nonce, aad, aad_len, ct, ct_len, calc);
ok = crypto_memeq_ct(calc, tag, 16);
crypto_memzero(calc, sizeof(calc));
if (!ok)
return -2;
chacha20_xor(key, nonce, 1, ct, pt, ct_len);
return 0;
}
case AEAD_AES256_GCM:
{
struct aes_gcm_ctx g;
int ret = aes_gcm_init(&g, key, key_len);
if (ret == 0)
ret = aes_gcm_decrypt(&g, nonce, nonce_len, aad, aad_len,
ct, ct_len, tag, pt) ? -2 : 0;
crypto_memzero(&g, sizeof(g));
return ret;
}
default:
break;
}
return -1;
}
Reference in new issue View Git Blame Copy Permalink