freebsd/freebsd-src
1
0 Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2026-11-07 04:36:52 -10:00
Code Issues Packages Projects Releases Wiki Activity

arm64: enable ossl accelerated AES-GCM on arm64

Browse Source 
Connect the baseline OpenSSL ARMv8 fused AES-GCM kernels to the OCF
ossl driver. Advertise the algorithm only when the system-wide capability set
includes both AES and PMULL.

Note that get_kernel_reg() is used to detect features such as PMULL
because the ossl device attaches before arm64 populates elf_hwcap when
ossl is built into the kernel or pre-loaded at boot.

Prefer OSSL over armv8crypto for AES-GCM sessions while retaining
armv8crypto as a fallback.

This provides about a 50% speedup for a Netflix ktls workload on a small
neoverse N1 board.

Reviewed by: markj
Constructive review feedback from: andrew, ngie
Differential Revision: https://reviews.freebsd.org/D60097
Sponsored by: Netflix
This commit is contained in:
Andrew Gallatin committed 2026-10-20 15:07:39 -04:00
1 parent f69099fc0f
commit ceb575553b
6 files changed
+126 -15
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

sys/conf/files.arm64
+8 -1
View File
@@ -197,14 +197,21 @@ aesv8-armx.o optional armv8crypto | ossl \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "aesv8-armx.o"
ghashv8-armx.o optional armv8crypto \
ghashv8-armx.o optional armv8crypto | ossl \
dependency "$S/crypto/openssl/aarch64/ghashv8-armx.S" \
compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "ghashv8-armx.o"
aes-gcm-armv8_64.o optional ossl \
dependency "$S/crypto/openssl/aarch64/aes-gcm-armv8_64.S" \
compile-with "${CC} -c ${CFLAGS:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \
no-implicit-rule \
clean "aes-gcm-armv8_64.o"
crypto/des/des_enc.c optional netsmb
crypto/openssl/ossl_aarch64.c optional ossl
crypto/openssl/ossl_aes_gcm.c optional ossl
crypto/openssl/aarch64/chacha-armv8.S optional ossl \
compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}"
crypto/openssl/aarch64/chacha-armv8-sve.S optional ossl \
sys/crypto/armv8/armv8_crypto.c
+5 -1
View File
@@ -209,7 +209,11 @@ armv8_crypto_probesession(device_t dev,
default:
return (EINVAL);
}
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
/*
* Prefer other crypto implementations (eg, ossl) to armv8_crypto
* by reducing the probe priority.
*/
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE - 50);
}
static int
sys/crypto/openssl/ossl.c
+1 -1
View File
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2020 Netflix, Inc
* Copyright (c) 2020-2026 Netflix, Inc
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
sys/crypto/openssl/ossl_aarch64.c
+24 -7
View File
@@ -1,6 +1,7 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Netflix, Inc
* Copyright (c) 2020 The FreeBSD Foundation
*
* This software was developed by Mitchell Horne <mhorne@FreeBSD.org>
@@ -29,9 +30,10 @@
*/
#include <sys/types.h>
#include <sys/stdint.h>
#include <machine/elf.h>
#include <machine/md_var.h>
#include <machine/armreg.h>
#include <machine/cpu.h>
#include <crypto/openssl/ossl.h>
#include <crypto/openssl/ossl_cipher.h>
@@ -41,9 +43,13 @@
* Feature bits defined in arm_arch.h
*/
unsigned int OPENSSL_armcap_P;
#define AES_PMULL_CAPABLE \
((OPENSSL_armcap_P & (ARMV8_AES | ARMV8_PMULL)) == \
(ARMV8_AES | ARMV8_PMULL))
ossl_cipher_setkey_t aes_v8_set_encrypt_key;
ossl_cipher_setkey_t aes_v8_set_decrypt_key;
ossl_cipher_setkey_t ossl_aes_gcm_setkey;
ossl_cipher_setkey_t vpaes_set_encrypt_key;
ossl_cipher_setkey_t vpaes_set_decrypt_key;
@@ -51,18 +57,23 @@ ossl_cipher_setkey_t vpaes_set_decrypt_key;
void
ossl_cpuid(struct ossl_softc *sc)
{
uint64_t isar0;
/* Use sanitized kernel features, available before elf_hwcap is set. */
get_kernel_reg(ID_AA64ISAR0_EL1, &isar0);
/* SHA features */
if ((elf_hwcap & HWCAP_SHA1) != 0)
if (ID_AA64ISAR0_SHA1_VAL(isar0) >= ID_AA64ISAR0_SHA1_BASE)
OPENSSL_armcap_P |= ARMV8_SHA1;
if ((elf_hwcap & HWCAP_SHA2) != 0)
if (ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_BASE)
OPENSSL_armcap_P |= ARMV8_SHA256;
if ((elf_hwcap & HWCAP_SHA512) != 0)
if (ID_AA64ISAR0_SHA2_VAL(isar0) >= ID_AA64ISAR0_SHA2_512)
OPENSSL_armcap_P |= ARMV8_SHA512;
/* AES features */
if ((elf_hwcap & HWCAP_AES) != 0)
if (ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE)
OPENSSL_armcap_P |= ARMV8_AES;
if ((elf_hwcap & HWCAP_PMULL) != 0)
if (ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_PMULL)
OPENSSL_armcap_P |= ARMV8_PMULL;
if ((OPENSSL_armcap_P & ARMV8_AES) == 0 &&
@@ -78,4 +89,10 @@ ossl_cpuid(struct ossl_softc *sc)
ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key;
ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key;
}
if (AES_PMULL_CAPABLE) {
ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey;
ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey;
sc->has_aes_gcm = true;
}
}
sys/crypto/openssl/ossl_aes_gcm.c
+84 -4
View File
@@ -10,10 +10,10 @@
*/
/*
* This file contains an AES-GCM wrapper implementation from OpenSSL, using
* AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from
* cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation
* for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
* This file contains AES-GCM wrapper implementations from OpenSSL, using
* AES-NI (x86), ARMv8 Crypto Extensions, or POWER8 Crypto Extensions. It
* makes use of a generic C implementation for partial blocks, ported from
* gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
*/
#include <sys/endian.h>
@@ -53,6 +53,86 @@ void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
size_t len);
#elif defined(__aarch64__)
#define AES_set_encrypt_key aes_v8_set_encrypt_key
#define AES_gcm_encrypt armv8_aes_gcm_encrypt
#define AES_gcm_decrypt armv8_aes_gcm_decrypt
#define AES_encrypt aes_v8_encrypt
#define AES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
#define GCM_init gcm_init_v8
#define GCM_gmult gcm_gmult_v8
#define GCM_ghash gcm_ghash_v8
int AES_set_encrypt_key(const unsigned char *key, int bits, void *ctx);
void AES_encrypt(const unsigned char *in, unsigned char *out, const void *ks);
void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
size_t blocks, const void *ks, const unsigned char *iv);
void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]);
void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]);
void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
size_t len);
typedef size_t armv8_gcm_kernel_t(const unsigned char *, uint64_t,
unsigned char *, uint64_t *, unsigned char *, const void *);
armv8_gcm_kernel_t aes_gcm_enc_128_kernel;
armv8_gcm_kernel_t aes_gcm_enc_192_kernel;
armv8_gcm_kernel_t aes_gcm_enc_256_kernel;
armv8_gcm_kernel_t aes_gcm_dec_128_kernel;
armv8_gcm_kernel_t aes_gcm_dec_192_kernel;
armv8_gcm_kernel_t aes_gcm_dec_256_kernel;
static size_t
armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
const void *key, unsigned char ivec[16], uint64_t Xi[2])
{
const struct ossl_aes_keysched *aes_key;
size_t bulk;
aes_key = key;
bulk = len - len % AES_BLOCK_LEN;
switch (aes_key->rounds) {
case 10:
(void)aes_gcm_enc_128_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
case 12:
(void)aes_gcm_enc_192_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
case 14:
(void)aes_gcm_enc_256_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
default:
return (0);
}
return (bulk);
}
static size_t
armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
const void *key, unsigned char ivec[16], uint64_t Xi[2])
{
const struct ossl_aes_keysched *aes_key;
size_t bulk;
aes_key = key;
bulk = len - len % AES_BLOCK_LEN;
switch (aes_key->rounds) {
case 10:
(void)aes_gcm_dec_128_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
case 12:
(void)aes_gcm_dec_192_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
case 14:
(void)aes_gcm_dec_256_kernel(in, bulk * 8, out, Xi, ivec, key);
break;
default:
return (0);
}
return (bulk);
}
#elif defined(__powerpc64__)
#define AES_set_encrypt_key aes_p8_set_encrypt_key
#define AES_gcm_encrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,1)
sys/modules/ossl/Makefile
+4 -1
View File
@@ -28,14 +28,17 @@ SRCS.arm= \
ossl_aes_gcm_neon.c
SRCS.aarch64= \
aes-gcm-armv8_64.S \
chacha-armv8.S \
chacha-armv8-sve.S \
ghashv8-armx.S \
poly1305-armv8.S \
sha1-armv8.S \
sha256-armv8.S \
sha512-armv8.S \
vpaes-armv8.S \
ossl_aarch64.c
ossl_aarch64.c \
ossl_aes_gcm.c
SRCS.amd64= \
aes-gcm-avx512.S \