LibCrypto: Implement AES by using x86 intrinsics

AK/CPUFeatures.cpp

@@ -44,6 +44,10 @@ CPUFeatures Detail::detect_cpu_features_uncached()
     if (cpuid7.ebx >> 29 & 1)
         result |= CPUFeatures::X86_SHA;
 #        endif
+#        if AK_CAN_CODEGEN_FOR_X86_AES
+    if (cpuid1.ecx >> 25 & 1)
+        result |= CPUFeatures::X86_AES;
+#        endif
 #    endif
 
     return result;

AK/CPUFeatures.h

@@ -20,11 +20,15 @@ enum class CPUFeatures : u64 {
     X86_SSE42 = 1ULL << 0,
 #    define AK_CAN_CODEGEN_FOR_X86_SHA 1
     X86_SHA = 1ULL << 1,
+#    define AK_CAN_CODEGEN_FOR_X86_AES 1
+    X86_AES = 1ULL << 2,
 #else
 #    define AK_CAN_CODEGEN_FOR_X86_SSE42 0
     X86_SSE42 = Invalid,
 #    define AK_CAN_CODEGEN_FOR_X86_SHA 0
     X86_SHA = Invalid,
+#    define AK_CAN_CODEGEN_FOR_X86_AES 0
+    X86_AES = Invalid,
 #endif
 };
 

Userland/Libraries/LibCrypto/Cipher/AES.cpp

@@ -4,6 +4,10 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/CPUFeatures.h>
+#include <AK/Platform.h>
+#include <AK/SIMD.h>
+#include <AK/SIMDExtras.h>
 #include <AK/StringBuilder.h>
 #include <LibCrypto/Cipher/AES.h>
 #include <LibCrypto/Cipher/AESTables.h>
@@ -41,7 +45,8 @@ ByteString AESCipherKey::to_byte_string() const
 }
 #endif
 
-void AESCipherKey::expand_encrypt_key(ReadonlyBytes user_key, size_t bits)
+template<>
+void AESCipherKey::expand_encrypt_key_impl<CPUFeatures::None>(ReadonlyBytes user_key, size_t bits)
 {
     u32* round_key;
     u32 temp;
@@ -152,7 +157,277 @@ void AESCipherKey::expand_encrypt_key(ReadonlyBytes user_key, size_t bits)
     }
 }
 
-void AESCipherKey::expand_decrypt_key(ReadonlyBytes user_key, size_t bits)
+#if AK_CAN_CODEGEN_FOR_X86_AES
+[[gnu::target("aes")]] static auto AESCipherKey_expand_encrypt_key_128_assist(auto const& a, auto const& b)
+{
+    AK::SIMD::i32x4 ta {};
+    AK::SIMD::i32x4 tb {};
+    AK::SIMD::i32x4 tc {};
+
+    static_assert(sizeof(a) == 16);
+    static_assert(sizeof(b) == 16);
+    static_assert(IsSame<decltype(a), decltype(b)>);
+
+    ta = bit_cast<AK::SIMD::i32x4>(a);
+    tb = bit_cast<AK::SIMD::i32x4>(b);
+    tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] };
+    tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
+    ta ^= tc;
+    tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
+    ta ^= tc;
+    tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
+    ta ^= tc;
+    ta ^= tb;
+    return bit_cast<RemoveReference<decltype(a)>>(ta);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_128(u32* round_key, ReadonlyBytes user_key)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    auto const* user_ptr = user_key.data();
+    illx2 ta {};
+    illx2 tb {};
+
+    ta = AK::SIMD::load_unaligned<illx2>(&user_ptr[0 * 16]);
+    AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x01);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[1 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x02);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[2 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x04);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[3 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x08);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[4 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x10);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[5 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x20);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[6 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x40);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[7 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x80);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[8 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x1b);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[9 * 4], ta);
+
+    tb = __builtin_ia32_aeskeygenassist128(ta, 0x36);
+    ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[10 * 4], ta);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192_assist(AK::SIMD::i32x4& a, AK::SIMD::i32x4& b, AK::SIMD::i32x4& c)
+{
+    AK::SIMD::i32x4 ta {};
+    AK::SIMD::i32x4 tb {};
+    AK::SIMD::i32x4 tc {};
+    AK::SIMD::i32x4 td {};
+
+    ta = bit_cast<AK::SIMD::i32x4>(a);
+    tb = bit_cast<AK::SIMD::i32x4>(b);
+    tc = bit_cast<AK::SIMD::i32x4>(c);
+    tb = AK::SIMD::i32x4 { tb[1], tb[1], tb[1], tb[1] };
+    td = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
+    ta ^= td;
+    td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] };
+    ta ^= td;
+    td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] };
+    ta ^= td;
+    ta ^= tb;
+    tb = AK::SIMD::i32x4 { ta[3], ta[3], ta[3], ta[3] };
+    td = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
+    tc ^= td;
+    tc ^= tb;
+    a = bit_cast<AK::SIMD::i32x4>(ta);
+    b = bit_cast<AK::SIMD::i32x4>(tb);
+    c = bit_cast<AK::SIMD::i32x4>(tc);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192(u32* round_key, ReadonlyBytes user_key)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    auto const* user_ptr = user_key.data();
+    AK::SIMD::i32x4 ta {};
+    AK::SIMD::i32x4 tb {};
+    AK::SIMD::i32x4 tc {};
+    AK::SIMD::i32x4 td {};
+
+    ta = AK::SIMD::load_unaligned<AK::SIMD::i32x4>(&user_ptr[0 * 16]);
+    tb = bit_cast<AK::SIMD::i32x4>(AK::SIMD::i64x2 { (i64)ByteReader::load64(&user_ptr[1 * 16]), 0 });
+    AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
+#    define ROUND(i1, i2, i3, k1, k2)                                                                  \
+        td = tb;                                                                                       \
+        tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), k1));    \
+        AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb);                                        \
+        AK::SIMD::store_unaligned(&round_key[i1 * 4], AK::SIMD::i32x4 { td[0], td[1], ta[0], ta[1] }); \
+        AK::SIMD::store_unaligned(&round_key[i2 * 4], AK::SIMD::i32x4 { ta[2], ta[3], tb[0], tb[1] }); \
+        tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), k2));    \
+        AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb);                                        \
+        AK::SIMD::store_unaligned(&round_key[i3 * 4], ta);
+
+    ROUND(1, 2, 3, 0x01, 0x02);
+    ROUND(4, 5, 6, 0x04, 0x08);
+    ROUND(7, 8, 9, 0x10, 0x20);
+    ROUND(10, 11, 12, 0x40, 0x80);
+    AK::SIMD::store_unaligned(&round_key[12 * 4], ta);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_a(AK::SIMD::i64x2& a, AK::SIMD::i64x2& b)
+{
+    AK::SIMD::i32x4 ta {};
+    AK::SIMD::i32x4 tb {};
+    AK::SIMD::i32x4 tc {};
+
+    ta = bit_cast<AK::SIMD::i32x4>(a);
+    tb = bit_cast<AK::SIMD::i32x4>(b);
+    tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] };
+    tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
+    ta ^= tc;
+    tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
+    ta ^= tc;
+    tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
+    ta ^= tc;
+    ta ^= tb;
+    a = bit_cast<AK::SIMD::i64x2>(ta);
+    b = bit_cast<AK::SIMD::i64x2>(tb);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_b(AK::SIMD::i64x2 const& a, AK::SIMD::i64x2& b)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    AK::SIMD::i32x4 tb {};
+    AK::SIMD::i32x4 tc {};
+    AK::SIMD::i32x4 td {};
+
+    tb = bit_cast<AK::SIMD::i32x4>(b);
+    tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(a), 0x00));
+    td = AK::SIMD::i32x4 { tc[2], tc[2], tc[2], tc[2] };
+    tc = AK::SIMD::i32x4 { 0, tb[0], tb[1], tb[2] };
+    tb ^= tc;
+    tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
+    tb ^= tc;
+    tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
+    tb ^= tc;
+    tb ^= td;
+    b = bit_cast<AK::SIMD::i64x2>(tb);
+}
+
+[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256(u32* round_key, ReadonlyBytes user_key)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    auto const* user_ptr = user_key.data();
+    AK::SIMD::i64x2 ta {};
+    AK::SIMD::i64x2 tb {};
+    AK::SIMD::i64x2 tc {};
+
+    ta = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&user_ptr[0 * 16]);
+    tb = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&user_ptr[1 * 16]);
+    AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
+    AK::SIMD::store_unaligned(&round_key[1 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x01));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[2 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[3 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x02));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[4 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[5 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x04));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[6 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[7 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x08));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[8 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[9 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x10));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[10 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[11 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x20));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[12 * 4], ta);
+
+    AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
+    AK::SIMD::store_unaligned(&round_key[13 * 4], tb);
+
+    tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x40));
+    AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
+    AK::SIMD::store_unaligned(&round_key[14 * 4], ta);
+}
+
+template<>
+[[gnu::target("aes")]] void AESCipherKey::expand_encrypt_key_impl<CPUFeatures::X86_AES>(ReadonlyBytes user_key, size_t bits)
+{
+    switch (bits) {
+    case 128:
+        m_rounds = 10;
+        AESCipherKey_expand_encrypt_key_128(m_rd_keys, user_key);
+        break;
+    case 192:
+        m_rounds = 12;
+        AESCipherKey_expand_encrypt_key_192(m_rd_keys, user_key);
+        break;
+    case 256:
+        m_rounds = 14;
+        AESCipherKey_expand_encrypt_key_256(m_rd_keys, user_key);
+        break;
+    }
+}
+
+template<>
+[[gnu::target("aes")]] void AESCipherKey::expand_decrypt_key_impl<CPUFeatures::X86_AES>(ReadonlyBytes user_key, size_t bits)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    u32* round_key = m_rd_keys;
+    expand_encrypt_key_impl<CPUFeatures::X86_AES>(user_key, bits);
+    for (size_t i_round = 1; i_round != m_rounds; ++i_round) {
+        AK::SIMD::i64x2 ta = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&round_key[i_round * 4]);
+        ta = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aesimc128(bit_cast<illx2>(ta)));
+        AK::SIMD::store_unaligned(&round_key[i_round * 4], ta);
+    }
+}
+#endif
+
+template<>
+void AESCipherKey::expand_decrypt_key_impl<CPUFeatures::None>(ReadonlyBytes user_key, size_t bits)
 {
     u32* round_key;
 
@@ -196,7 +471,30 @@ void AESCipherKey::expand_decrypt_key(ReadonlyBytes user_key, size_t bits)
     }
 }
 
-void AESCipher::encrypt_block(AESCipherBlock const& in, AESCipherBlock& out)
+decltype(AESCipherKey::expand_encrypt_key_dispatched) AESCipherKey::expand_encrypt_key_dispatched = [] {
+    CPUFeatures features = detect_cpu_features();
+
+    if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
+        if (has_flag(features, CPUFeatures::X86_AES))
+            return &AESCipherKey::expand_encrypt_key_impl<CPUFeatures::X86_AES>;
+    }
+
+    return &AESCipherKey::expand_encrypt_key_impl<CPUFeatures::None>;
+}();
+
+decltype(AESCipherKey::expand_decrypt_key_dispatched) AESCipherKey::expand_decrypt_key_dispatched = [] {
+    CPUFeatures features = detect_cpu_features();
+
+    if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
+        if (has_flag(features, CPUFeatures::X86_AES))
+            return &AESCipherKey::expand_decrypt_key_impl<CPUFeatures::X86_AES>;
+    }
+
+    return &AESCipherKey::expand_decrypt_key_impl<CPUFeatures::None>;
+}();
+
+template<>
+void AESCipher::encrypt_block_impl<CPUFeatures::None>(AESCipherBlock const& in, AESCipherBlock& out)
 {
     u32 s0, s1, s2, s3, t0, t1, t2, t3;
     size_t r { 0 };
@@ -285,7 +583,33 @@ void AESCipher::encrypt_block(AESCipherBlock const& in, AESCipherBlock& out)
     // clang-format on
 }
 
-void AESCipher::decrypt_block(AESCipherBlock const& in, AESCipherBlock& out)
+#if AK_CAN_CODEGEN_FOR_X86_AES
+template<>
+[[gnu::target("aes")]] void AESCipher::encrypt_block_impl<CPUFeatures::X86_AES>(AESCipherBlock const& in, AESCipherBlock& out)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    AESCipherKey const& key = m_key;
+    auto round_keys = key.round_keys();
+    auto n_rounds = static_cast<int>(key.rounds());
+    auto input_ptr = in.bytes().data();
+    auto output_ptr = out.bytes().data();
+
+    auto value = AK::SIMD::load_unaligned<illx2>(input_ptr);
+    auto round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(0 + 0) * 4]);
+    value ^= round_key;
+    for (int i_round = 0; i_round != n_rounds - 1; ++i_round) {
+        round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(1 + i_round) * 4]);
+        value = __builtin_ia32_aesenc128(value, round_key);
+    }
+    round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(1 + (n_rounds - 1)) * 4]);
+    value = __builtin_ia32_aesenclast128(value, round_key);
+    AK::SIMD::store_unaligned(output_ptr, value);
+}
+#endif
+
+template<>
+void AESCipher::decrypt_block_impl<CPUFeatures::None>(AESCipherBlock const& in, AESCipherBlock& out)
 {
     u32 s0, s1, s2, s3, t0, t1, t2, t3;
     size_t r { 0 };
@@ -374,6 +698,54 @@ void AESCipher::decrypt_block(AESCipherBlock const& in, AESCipherBlock& out)
     // clang-format on
 }
 
+#if AK_CAN_CODEGEN_FOR_X86_AES
+template<>
+[[gnu::target("aes")]] void AESCipher::decrypt_block_impl<CPUFeatures::X86_AES>(AESCipherBlock const& in, AESCipherBlock& out)
+{
+    using illx2 = signed long long int __attribute__((vector_size(16)));
+
+    AESCipherKey const& key = m_key;
+    auto round_keys = key.round_keys();
+    auto n_rounds = static_cast<int>(key.rounds());
+    auto input_ptr = in.bytes().data();
+    auto output_ptr = out.bytes().data();
+
+    auto value = AK::SIMD::load_unaligned<illx2>(input_ptr);
+    auto round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[n_rounds * 4]);
+    value ^= round_key;
+    for (int i_round = 0; i_round != n_rounds - 1; ++i_round) {
+        int const idx = (n_rounds - 1) - i_round;
+        round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[idx * 4]);
+        value = __builtin_ia32_aesdec128(value, round_key);
+    }
+    round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[0]);
+    value = __builtin_ia32_aesdeclast128(value, round_key);
+    AK::SIMD::store_unaligned(output_ptr, value);
+}
+#endif
+
+decltype(AESCipher::encrypt_block_dispatched) AESCipher::encrypt_block_dispatched = [] {
+    CPUFeatures features = detect_cpu_features();
+
+    if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
+        if (has_flag(features, CPUFeatures::X86_AES))
+            return &AESCipher::encrypt_block_impl<CPUFeatures::X86_AES>;
+    }
+
+    return &AESCipher::encrypt_block_impl<CPUFeatures::None>;
+}();
+
+decltype(AESCipher::decrypt_block_dispatched) AESCipher::decrypt_block_dispatched = [] {
+    CPUFeatures features = detect_cpu_features();
+
+    if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
+        if (has_flag(features, CPUFeatures::X86_AES))
+            return &AESCipher::decrypt_block_impl<CPUFeatures::X86_AES>;
+    }
+
+    return &AESCipher::decrypt_block_impl<CPUFeatures::None>;
+}();
+
 void AESCipherBlock::overwrite(ReadonlyBytes bytes)
 {
     auto data = bytes.data();

Userland/Libraries/LibCrypto/Cipher/AES.h

@@ -7,6 +7,7 @@
 
 #pragma once
 
+#include <AK/CPUFeatures.h>
 #include <AK/Vector.h>
 #include <LibCrypto/Cipher/Cipher.h>
 #include <LibCrypto/Cipher/Mode/CBC.h>
@@ -59,8 +60,8 @@ private:
 
 struct AESCipherKey : public CipherKey {
     virtual ReadonlyBytes bytes() const override { return ReadonlyBytes { m_rd_keys, sizeof(m_rd_keys) }; }
-    virtual void expand_encrypt_key(ReadonlyBytes user_key, size_t bits) override;
-    virtual void expand_decrypt_key(ReadonlyBytes user_key, size_t bits) override;
+    virtual void expand_encrypt_key(ReadonlyBytes user_key, size_t bits) override { return (this->*expand_encrypt_key_dispatched)(user_key, bits); }
+    virtual void expand_decrypt_key(ReadonlyBytes user_key, size_t bits) override { return (this->*expand_decrypt_key_dispatched)(user_key, bits); }
     static bool is_valid_key_size(size_t bits) { return bits == 128 || bits == 192 || bits == 256; }
 
 #ifndef KERNEL
@@ -93,6 +94,14 @@ protected:
     }
 
 private:
+    template<CPUFeatures>
+    void expand_encrypt_key_impl(ReadonlyBytes user_key, size_t bits);
+    template<CPUFeatures>
+    void expand_decrypt_key_impl(ReadonlyBytes user_key, size_t bits);
+
+    static void (AESCipherKey::*const expand_encrypt_key_dispatched)(ReadonlyBytes user_key, size_t bits);
+    static void (AESCipherKey::*const expand_decrypt_key_dispatched)(ReadonlyBytes user_key, size_t bits);
+
     static constexpr size_t MAX_ROUND_COUNT = 14;
     u32 m_rd_keys[(MAX_ROUND_COUNT + 1) * 4] { 0 };
     size_t m_rounds;
@@ -116,8 +125,8 @@ public:
     virtual AESCipherKey const& key() const override { return m_key; }
     virtual AESCipherKey& key() override { return m_key; }
 
-    virtual void encrypt_block(BlockType const& in, BlockType& out) override;
-    virtual void decrypt_block(BlockType const& in, BlockType& out) override;
+    virtual void encrypt_block(BlockType const& in, BlockType& out) override { return (this->*encrypt_block_dispatched)(in, out); }
+    virtual void decrypt_block(BlockType const& in, BlockType& out) override { return (this->*decrypt_block_dispatched)(in, out); }
 
 #ifndef KERNEL
     virtual ByteString class_name() const override
@@ -128,6 +137,15 @@ public:
 
 protected:
     AESCipherKey m_key;
+
+private:
+    template<CPUFeatures>
+    void encrypt_block_impl(BlockType const& in, BlockType& out);
+    template<CPUFeatures>
+    void decrypt_block_impl(BlockType const& in, BlockType& out);
+
+    static void (AESCipher::*const encrypt_block_dispatched)(BlockType const& in, BlockType& out);
+    static void (AESCipher::*const decrypt_block_dispatched)(BlockType const& in, BlockType& out);
 };
 
 }