diff --git a/AK/CPUFeatures.cpp b/AK/CPUFeatures.cpp index 8b6cb1ff6cc..02b9bdcce86 100644 --- a/AK/CPUFeatures.cpp +++ b/AK/CPUFeatures.cpp @@ -44,6 +44,10 @@ CPUFeatures Detail::detect_cpu_features_uncached() if (cpuid7.ebx >> 29 & 1) result |= CPUFeatures::X86_SHA; # endif +# if AK_CAN_CODEGEN_FOR_X86_AES + if (cpuid1.ecx >> 25 & 1) + result |= CPUFeatures::X86_AES; +# endif # endif return result; diff --git a/AK/CPUFeatures.h b/AK/CPUFeatures.h index 9a4b7d78892..f0369fb4cfd 100644 --- a/AK/CPUFeatures.h +++ b/AK/CPUFeatures.h @@ -20,11 +20,15 @@ enum class CPUFeatures : u64 { X86_SSE42 = 1ULL << 0, # define AK_CAN_CODEGEN_FOR_X86_SHA 1 X86_SHA = 1ULL << 1, +# define AK_CAN_CODEGEN_FOR_X86_AES 1 + X86_AES = 1ULL << 2, #else # define AK_CAN_CODEGEN_FOR_X86_SSE42 0 X86_SSE42 = Invalid, # define AK_CAN_CODEGEN_FOR_X86_SHA 0 X86_SHA = Invalid, +# define AK_CAN_CODEGEN_FOR_X86_AES 0 + X86_AES = Invalid, #endif }; diff --git a/Userland/Libraries/LibCrypto/Cipher/AES.cpp b/Userland/Libraries/LibCrypto/Cipher/AES.cpp index 1c63e5abb17..24aa286cff2 100644 --- a/Userland/Libraries/LibCrypto/Cipher/AES.cpp +++ b/Userland/Libraries/LibCrypto/Cipher/AES.cpp @@ -4,6 +4,10 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include +#include +#include +#include #include #include #include @@ -41,7 +45,8 @@ ByteString AESCipherKey::to_byte_string() const } #endif -void AESCipherKey::expand_encrypt_key(ReadonlyBytes user_key, size_t bits) +template<> +void AESCipherKey::expand_encrypt_key_impl(ReadonlyBytes user_key, size_t bits) { u32* round_key; u32 temp; @@ -152,7 +157,277 @@ void AESCipherKey::expand_encrypt_key(ReadonlyBytes user_key, size_t bits) } } -void AESCipherKey::expand_decrypt_key(ReadonlyBytes user_key, size_t bits) +#if AK_CAN_CODEGEN_FOR_X86_AES +[[gnu::target("aes")]] static auto AESCipherKey_expand_encrypt_key_128_assist(auto const& a, auto const& b) +{ + AK::SIMD::i32x4 ta {}; + AK::SIMD::i32x4 tb {}; + AK::SIMD::i32x4 tc {}; + + static_assert(sizeof(a) == 16); + static_assert(sizeof(b) == 16); + static_assert(IsSame); + + ta = bit_cast(a); + tb = bit_cast(b); + tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] }; + tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] }; + ta ^= tc; + tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] }; + ta ^= tc; + tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] }; + ta ^= tc; + ta ^= tb; + return bit_cast>(ta); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_128(u32* round_key, ReadonlyBytes user_key) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + auto const* user_ptr = user_key.data(); + illx2 ta {}; + illx2 tb {}; + + ta = AK::SIMD::load_unaligned(&user_ptr[0 * 16]); + AK::SIMD::store_unaligned(&round_key[0 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x01); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[1 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x02); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[2 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x04); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[3 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x08); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[4 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x10); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[5 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x20); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[6 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x40); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[7 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x80); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[8 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x1b); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[9 * 4], ta); + + tb = __builtin_ia32_aeskeygenassist128(ta, 0x36); + ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb); + AK::SIMD::store_unaligned(&round_key[10 * 4], ta); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192_assist(AK::SIMD::i32x4& a, AK::SIMD::i32x4& b, AK::SIMD::i32x4& c) +{ + AK::SIMD::i32x4 ta {}; + AK::SIMD::i32x4 tb {}; + AK::SIMD::i32x4 tc {}; + AK::SIMD::i32x4 td {}; + + ta = bit_cast(a); + tb = bit_cast(b); + tc = bit_cast(c); + tb = AK::SIMD::i32x4 { tb[1], tb[1], tb[1], tb[1] }; + td = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] }; + ta ^= td; + td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] }; + ta ^= td; + td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] }; + ta ^= td; + ta ^= tb; + tb = AK::SIMD::i32x4 { ta[3], ta[3], ta[3], ta[3] }; + td = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] }; + tc ^= td; + tc ^= tb; + a = bit_cast(ta); + b = bit_cast(tb); + c = bit_cast(tc); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192(u32* round_key, ReadonlyBytes user_key) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + auto const* user_ptr = user_key.data(); + AK::SIMD::i32x4 ta {}; + AK::SIMD::i32x4 tb {}; + AK::SIMD::i32x4 tc {}; + AK::SIMD::i32x4 td {}; + + ta = AK::SIMD::load_unaligned(&user_ptr[0 * 16]); + tb = bit_cast(AK::SIMD::i64x2 { (i64)ByteReader::load64(&user_ptr[1 * 16]), 0 }); + AK::SIMD::store_unaligned(&round_key[0 * 4], ta); +# define ROUND(i1, i2, i3, k1, k2) \ + td = tb; \ + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), k1)); \ + AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb); \ + AK::SIMD::store_unaligned(&round_key[i1 * 4], AK::SIMD::i32x4 { td[0], td[1], ta[0], ta[1] }); \ + AK::SIMD::store_unaligned(&round_key[i2 * 4], AK::SIMD::i32x4 { ta[2], ta[3], tb[0], tb[1] }); \ + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), k2)); \ + AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb); \ + AK::SIMD::store_unaligned(&round_key[i3 * 4], ta); + + ROUND(1, 2, 3, 0x01, 0x02); + ROUND(4, 5, 6, 0x04, 0x08); + ROUND(7, 8, 9, 0x10, 0x20); + ROUND(10, 11, 12, 0x40, 0x80); + AK::SIMD::store_unaligned(&round_key[12 * 4], ta); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_a(AK::SIMD::i64x2& a, AK::SIMD::i64x2& b) +{ + AK::SIMD::i32x4 ta {}; + AK::SIMD::i32x4 tb {}; + AK::SIMD::i32x4 tc {}; + + ta = bit_cast(a); + tb = bit_cast(b); + tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] }; + tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] }; + ta ^= tc; + tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] }; + ta ^= tc; + tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] }; + ta ^= tc; + ta ^= tb; + a = bit_cast(ta); + b = bit_cast(tb); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_b(AK::SIMD::i64x2 const& a, AK::SIMD::i64x2& b) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + AK::SIMD::i32x4 tb {}; + AK::SIMD::i32x4 tc {}; + AK::SIMD::i32x4 td {}; + + tb = bit_cast(b); + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(a), 0x00)); + td = AK::SIMD::i32x4 { tc[2], tc[2], tc[2], tc[2] }; + tc = AK::SIMD::i32x4 { 0, tb[0], tb[1], tb[2] }; + tb ^= tc; + tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] }; + tb ^= tc; + tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] }; + tb ^= tc; + tb ^= td; + b = bit_cast(tb); +} + +[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256(u32* round_key, ReadonlyBytes user_key) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + auto const* user_ptr = user_key.data(); + AK::SIMD::i64x2 ta {}; + AK::SIMD::i64x2 tb {}; + AK::SIMD::i64x2 tc {}; + + ta = AK::SIMD::load_unaligned(&user_ptr[0 * 16]); + tb = AK::SIMD::load_unaligned(&user_ptr[1 * 16]); + AK::SIMD::store_unaligned(&round_key[0 * 4], ta); + AK::SIMD::store_unaligned(&round_key[1 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x01)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[2 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[3 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x02)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[4 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[5 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x04)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[6 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[7 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x08)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[8 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[9 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x10)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[10 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[11 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x20)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[12 * 4], ta); + + AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb); + AK::SIMD::store_unaligned(&round_key[13 * 4], tb); + + tc = bit_cast(__builtin_ia32_aeskeygenassist128(bit_cast(tb), 0x40)); + AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc); + AK::SIMD::store_unaligned(&round_key[14 * 4], ta); +} + +template<> +[[gnu::target("aes")]] void AESCipherKey::expand_encrypt_key_impl(ReadonlyBytes user_key, size_t bits) +{ + switch (bits) { + case 128: + m_rounds = 10; + AESCipherKey_expand_encrypt_key_128(m_rd_keys, user_key); + break; + case 192: + m_rounds = 12; + AESCipherKey_expand_encrypt_key_192(m_rd_keys, user_key); + break; + case 256: + m_rounds = 14; + AESCipherKey_expand_encrypt_key_256(m_rd_keys, user_key); + break; + } +} + +template<> +[[gnu::target("aes")]] void AESCipherKey::expand_decrypt_key_impl(ReadonlyBytes user_key, size_t bits) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + u32* round_key = m_rd_keys; + expand_encrypt_key_impl(user_key, bits); + for (size_t i_round = 1; i_round != m_rounds; ++i_round) { + AK::SIMD::i64x2 ta = AK::SIMD::load_unaligned(&round_key[i_round * 4]); + ta = bit_cast(__builtin_ia32_aesimc128(bit_cast(ta))); + AK::SIMD::store_unaligned(&round_key[i_round * 4], ta); + } +} +#endif + +template<> +void AESCipherKey::expand_decrypt_key_impl(ReadonlyBytes user_key, size_t bits) { u32* round_key; @@ -196,7 +471,30 @@ void AESCipherKey::expand_decrypt_key(ReadonlyBytes user_key, size_t bits) } } -void AESCipher::encrypt_block(AESCipherBlock const& in, AESCipherBlock& out) +decltype(AESCipherKey::expand_encrypt_key_dispatched) AESCipherKey::expand_encrypt_key_dispatched = [] { + CPUFeatures features = detect_cpu_features(); + + if constexpr (is_valid_feature(CPUFeatures::X86_AES)) { + if (has_flag(features, CPUFeatures::X86_AES)) + return &AESCipherKey::expand_encrypt_key_impl; + } + + return &AESCipherKey::expand_encrypt_key_impl; +}(); + +decltype(AESCipherKey::expand_decrypt_key_dispatched) AESCipherKey::expand_decrypt_key_dispatched = [] { + CPUFeatures features = detect_cpu_features(); + + if constexpr (is_valid_feature(CPUFeatures::X86_AES)) { + if (has_flag(features, CPUFeatures::X86_AES)) + return &AESCipherKey::expand_decrypt_key_impl; + } + + return &AESCipherKey::expand_decrypt_key_impl; +}(); + +template<> +void AESCipher::encrypt_block_impl(AESCipherBlock const& in, AESCipherBlock& out) { u32 s0, s1, s2, s3, t0, t1, t2, t3; size_t r { 0 }; @@ -285,7 +583,33 @@ void AESCipher::encrypt_block(AESCipherBlock const& in, AESCipherBlock& out) // clang-format on } -void AESCipher::decrypt_block(AESCipherBlock const& in, AESCipherBlock& out) +#if AK_CAN_CODEGEN_FOR_X86_AES +template<> +[[gnu::target("aes")]] void AESCipher::encrypt_block_impl(AESCipherBlock const& in, AESCipherBlock& out) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + AESCipherKey const& key = m_key; + auto round_keys = key.round_keys(); + auto n_rounds = static_cast(key.rounds()); + auto input_ptr = in.bytes().data(); + auto output_ptr = out.bytes().data(); + + auto value = AK::SIMD::load_unaligned(input_ptr); + auto round_key = AK::SIMD::load_unaligned(&round_keys[(0 + 0) * 4]); + value ^= round_key; + for (int i_round = 0; i_round != n_rounds - 1; ++i_round) { + round_key = AK::SIMD::load_unaligned(&round_keys[(1 + i_round) * 4]); + value = __builtin_ia32_aesenc128(value, round_key); + } + round_key = AK::SIMD::load_unaligned(&round_keys[(1 + (n_rounds - 1)) * 4]); + value = __builtin_ia32_aesenclast128(value, round_key); + AK::SIMD::store_unaligned(output_ptr, value); +} +#endif + +template<> +void AESCipher::decrypt_block_impl(AESCipherBlock const& in, AESCipherBlock& out) { u32 s0, s1, s2, s3, t0, t1, t2, t3; size_t r { 0 }; @@ -374,6 +698,54 @@ void AESCipher::decrypt_block(AESCipherBlock const& in, AESCipherBlock& out) // clang-format on } +#if AK_CAN_CODEGEN_FOR_X86_AES +template<> +[[gnu::target("aes")]] void AESCipher::decrypt_block_impl(AESCipherBlock const& in, AESCipherBlock& out) +{ + using illx2 = signed long long int __attribute__((vector_size(16))); + + AESCipherKey const& key = m_key; + auto round_keys = key.round_keys(); + auto n_rounds = static_cast(key.rounds()); + auto input_ptr = in.bytes().data(); + auto output_ptr = out.bytes().data(); + + auto value = AK::SIMD::load_unaligned(input_ptr); + auto round_key = AK::SIMD::load_unaligned(&round_keys[n_rounds * 4]); + value ^= round_key; + for (int i_round = 0; i_round != n_rounds - 1; ++i_round) { + int const idx = (n_rounds - 1) - i_round; + round_key = AK::SIMD::load_unaligned(&round_keys[idx * 4]); + value = __builtin_ia32_aesdec128(value, round_key); + } + round_key = AK::SIMD::load_unaligned(&round_keys[0]); + value = __builtin_ia32_aesdeclast128(value, round_key); + AK::SIMD::store_unaligned(output_ptr, value); +} +#endif + +decltype(AESCipher::encrypt_block_dispatched) AESCipher::encrypt_block_dispatched = [] { + CPUFeatures features = detect_cpu_features(); + + if constexpr (is_valid_feature(CPUFeatures::X86_AES)) { + if (has_flag(features, CPUFeatures::X86_AES)) + return &AESCipher::encrypt_block_impl; + } + + return &AESCipher::encrypt_block_impl; +}(); + +decltype(AESCipher::decrypt_block_dispatched) AESCipher::decrypt_block_dispatched = [] { + CPUFeatures features = detect_cpu_features(); + + if constexpr (is_valid_feature(CPUFeatures::X86_AES)) { + if (has_flag(features, CPUFeatures::X86_AES)) + return &AESCipher::decrypt_block_impl; + } + + return &AESCipher::decrypt_block_impl; +}(); + void AESCipherBlock::overwrite(ReadonlyBytes bytes) { auto data = bytes.data(); diff --git a/Userland/Libraries/LibCrypto/Cipher/AES.h b/Userland/Libraries/LibCrypto/Cipher/AES.h index 835188c17cc..ef87043b603 100644 --- a/Userland/Libraries/LibCrypto/Cipher/AES.h +++ b/Userland/Libraries/LibCrypto/Cipher/AES.h @@ -7,6 +7,7 @@ #pragma once +#include #include #include #include @@ -59,8 +60,8 @@ private: struct AESCipherKey : public CipherKey { virtual ReadonlyBytes bytes() const override { return ReadonlyBytes { m_rd_keys, sizeof(m_rd_keys) }; } - virtual void expand_encrypt_key(ReadonlyBytes user_key, size_t bits) override; - virtual void expand_decrypt_key(ReadonlyBytes user_key, size_t bits) override; + virtual void expand_encrypt_key(ReadonlyBytes user_key, size_t bits) override { return (this->*expand_encrypt_key_dispatched)(user_key, bits); } + virtual void expand_decrypt_key(ReadonlyBytes user_key, size_t bits) override { return (this->*expand_decrypt_key_dispatched)(user_key, bits); } static bool is_valid_key_size(size_t bits) { return bits == 128 || bits == 192 || bits == 256; } #ifndef KERNEL @@ -93,6 +94,14 @@ protected: } private: + template + void expand_encrypt_key_impl(ReadonlyBytes user_key, size_t bits); + template + void expand_decrypt_key_impl(ReadonlyBytes user_key, size_t bits); + + static void (AESCipherKey::*const expand_encrypt_key_dispatched)(ReadonlyBytes user_key, size_t bits); + static void (AESCipherKey::*const expand_decrypt_key_dispatched)(ReadonlyBytes user_key, size_t bits); + static constexpr size_t MAX_ROUND_COUNT = 14; u32 m_rd_keys[(MAX_ROUND_COUNT + 1) * 4] { 0 }; size_t m_rounds; @@ -116,8 +125,8 @@ public: virtual AESCipherKey const& key() const override { return m_key; } virtual AESCipherKey& key() override { return m_key; } - virtual void encrypt_block(BlockType const& in, BlockType& out) override; - virtual void decrypt_block(BlockType const& in, BlockType& out) override; + virtual void encrypt_block(BlockType const& in, BlockType& out) override { return (this->*encrypt_block_dispatched)(in, out); } + virtual void decrypt_block(BlockType const& in, BlockType& out) override { return (this->*decrypt_block_dispatched)(in, out); } #ifndef KERNEL virtual ByteString class_name() const override @@ -128,6 +137,15 @@ public: protected: AESCipherKey m_key; + +private: + template + void encrypt_block_impl(BlockType const& in, BlockType& out); + template + void decrypt_block_impl(BlockType const& in, BlockType& out); + + static void (AESCipher::*const encrypt_block_dispatched)(BlockType const& in, BlockType& out); + static void (AESCipher::*const decrypt_block_dispatched)(BlockType const& in, BlockType& out); }; }